From 48b40bd620319c80cd1196058d7e241a1b6f9307 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Mon, 17 Aug 2009 15:58:06 -0300 Subject: [PATCH] renamed x method of selectors to select --- docs/intro/overview.rst | 6 +- docs/intro/tutorial.rst | 46 ++++++------- docs/ref/selectors.rst | 22 +++---- docs/ref/spiders.rst | 12 ++-- docs/topics/firebug.rst | 8 +-- docs/topics/selectors.rst | 36 +++++------ docs/topics/shell.rst | 6 +- .../googledir/spiders/google_directory.py | 8 +-- scrapy/contrib/linkextractors/image.py | 11 ++-- scrapy/contrib/linkextractors/sgml.py | 3 +- scrapy/contrib/loader/__init__.py | 2 +- scrapy/contrib/spiders/feed.py | 4 +- scrapy/tests/test_utils_iterators.py | 22 +++---- scrapy/tests/test_xpath.py | 64 +++++++++---------- scrapy/utils/decorator.py | 24 +++++++ scrapy/utils/iterators.py | 2 +- scrapy/xpath/selector.py | 22 +++++-- 17 files changed, 168 insertions(+), 130 deletions(-) create mode 100644 scrapy/utils/decorator.py diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index a7d6a53c1..55605a5e8 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -132,9 +132,9 @@ Finally, here's the spider code:: torrent = ScrapedItem() torrent.url = response.url - torrent.name = x.x("//h1/text()").extract() - torrent.description = x.x("//div[@id='description']").extract() - torrent.size = x.x("//div[@id='info-left']/p[2]/text()[2]").extract() + torrent.name = x.select("//h1/text()").extract() + torrent.description = x.select("//div[@id='description']").extract() + torrent.size = x.select("//div[@id='info-left']/p[2]/text()[2]").extract() return [torrent] diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 90de0b27c..21848591e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -264,19 +264,19 @@ The shell also instantiates two selectors, one for HTML (in the ``hxs`` variable) and one for XML (in the ``xxs`` variable)with this response. So let's try them:: - In [1]: hxs.x('/html/head/title') + In [1]: hxs.select('/html/head/title') Out[1]: [] - In [2]: hxs.x('/html/head/title').extract() + In [2]: hxs.select('/html/head/title').extract() Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: hxs.x('/html/head/title/text()') + In [3]: hxs.select('/html/head/title/text()') Out[3]: [] - In [4]: hxs.x('/html/head/title/text()').extract() + In [4]: hxs.select('/html/head/title/text()').extract() Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: hxs.x('/html/head/title/text()').re('(\w+):') + In [5]: hxs.select('/html/head/title/text()').re('(\w+):') Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data @@ -296,29 +296,29 @@ is inside a ``
    `` element, in fact the *second* ``
      `` element. So we can select each ``
    • `` element belonging to the sites list with this code:: - hxs.x('//ul[2]/li') + hxs.select('//ul[2]/li') And from them, the sites descriptions:: - hxs.x('//ul[2]/li/text()').extract() + hxs.select('//ul[2]/li/text()').extract() The sites titles:: - hxs.x('//ul[2]/li/a/text()').extract() + hxs.select('//ul[2]/li/a/text()').extract() And the sites links:: - hxs.x('//ul[2]/li/a/@href').extract() + hxs.select('//ul[2]/li/a/@href').extract() -As we said before, each ``x()`` call returns a list of selectors, so we can -concatenate further ``x()`` calls to dig deeper into a node. We are going to use +As we said before, each ``select()`` call returns a list of selectors, so we can +concatenate further ``select()`` calls to dig deeper into a node. We are going to use that property here, so:: - sites = hxs.x('//ul[2]/li') + sites = hxs.select('//ul[2]/li') for site in sites: - title = site.x('a/text()').extract() - link = site.x('a/@href').extract() - desc = site.x('text()').extract() + title = site.select('a/text()').extract() + link = site.select('a/@href').extract() + desc = site.select('text()').extract() print title, link, desc .. note:: @@ -342,11 +342,11 @@ Let's add this code to our spider:: def parse(self, response): hxs = HtmlXPathSelector(response) - sites = hxs.x('//ul[2]/li') + sites = hxs.select('//ul[2]/li') for site in sites: - title = site.x('a/text()').extract() - link = site.x('a/@href').extract() - desc = site.x('text()').extract() + title = site.select('a/text()').extract() + link = site.select('a/@href').extract() + desc = site.select('text()').extract() print title, link, desc return [] @@ -375,13 +375,13 @@ should be like this:: def parse(self, response): hxs = HtmlXPathSelector(response) - sites = hxs.x('//ul[2]/li') + sites = hxs.select('//ul[2]/li') items = [] for site in sites: item = DmozItem() - item.title = site.x('a/text()').extract() - item.link = site.x('a/@href').extract() - item.desc = site.x('text()').extract() + item.title = site.select('a/text()').extract() + item.link = site.select('a/@href').extract() + item.desc = site.select('text()').extract() items.append(item) return items diff --git a/docs/ref/selectors.rst b/docs/ref/selectors.rst index 9ebbb3110..5be263cdc 100644 --- a/docs/ref/selectors.rst +++ b/docs/ref/selectors.rst @@ -31,7 +31,7 @@ XPathSelector objects XPathSelector Methods --------------------- -.. method:: XPathSelector.x(xpath) +.. method:: XPathSelector.select(xpath) Apply the given XPath relative to this XPathSelector and return a list of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) with @@ -84,7 +84,7 @@ XPathSelectorList objects XPathSelectorList Methods ------------------------- -.. method:: XPathSelectorList.x(xpath) +.. method:: XPathSelectorList.select(xpath) Call the :meth:`XPathSelector.re` method for all :class:`XPathSelector` objects in this list and return their results flattened, as new @@ -136,27 +136,27 @@ instanced with a :class:`~scrapy.http.Response` object like this:: 1. Select all ``

      `` elements from a HTML response body, returning a list of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object):: - x.x("//h1") + x.select("//h1") 2. Extract the text of all ``

      `` elements from a HTML response body, returning a list of unicode strings:: - x.x("//h1").extract() # this includes the h1 tag - x.x("//h1/text()").extract() # this excludes the h1 tag + x.select("//h1").extract() # this includes the h1 tag + x.select("//h1/text()").extract() # this excludes the h1 tag 3. Iterate over all ``

      `` tags and print their class attribute:: - for node in x.x("//p"): - ... print node.x("@href") + for node in x.select("//p"): + ... print node.select("@href") 4. Extract textual data from all ``

      `` tags without entities, as a list of unicode strings:: - x.x("//p/text()").extract_unquoted() + x.select("//p/text()").extract_unquoted() # the following line is wrong. extract_unquoted() should only be used # with textual XPathSelectors - x.x("//p").extract_unquoted() # it may work but output is unpredictable + x.select("//p").extract_unquoted() # it may work but output is unpredictable XmlXPathSelector objects ======================== @@ -178,12 +178,12 @@ instanced with a :class:`~scrapy.http.Response` object like this:: 1. Select all ```` elements from a XML response body, returning a list of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object):: - x.x("//h1") + x.select("//h1") 2. Extract all prices from a `Google Base XML feed`_ which requires registering a namespace:: x.register_namespace("g", "http://base.google.com/ns/1.0") - x.x("//g:price").extract() + x.select("//g:price").extract() .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/docs/ref/spiders.rst b/docs/ref/spiders.rst index 950e1a361..2011562ed 100644 --- a/docs/ref/spiders.rst +++ b/docs/ref/spiders.rst @@ -189,9 +189,9 @@ Let's now take a look at an example CrawlSpider with rules:: hxs = HtmlXPathSelector(response) item = ScrapedItem() - item.id = hxs.x('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item.name = hxs.x('//td[@id="item_name"]/text()').extract() - item.description = hxs.x('//td[@id="item_description"]/text()').extract() + item.id = hxs.select('//td[@id="item_id"]/text()').re(r'ID: (\d+)') + item.name = hxs.select('//td[@id="item_name"]/text()').extract() + item.description = hxs.select('//td[@id="item_description"]/text()').extract() return [item] SPIDER = MySpider() @@ -307,9 +307,9 @@ These spiders are pretty easy to use, let's have at one example:: log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) item = ScrapedItem() - item.id = node.x('@id').extract() - item.name = node.x('name').extract() - item.description = node.x('description').extract() + item.id = node.select('@id').extract() + item.name = node.select('name').extract() + item.description = node.select('description').extract() return item SPIDER = MySpider() diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 37a5d4191..bb85e3946 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -144,13 +144,13 @@ Finally, we can write our ``parse_category()`` method:: hxs = HtmlXPathSelector(response) # The path to website links in directory page - links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') for link in links: item = ScrapedItem() - item.name = link.x('a/text()').extract() - item.url = link.x('a/@href').extract() - item.description = link.x('font[2]/text()').extract() + item.name = link.select('a/text()').extract() + item.url = link.select('a/@href').extract() + item.description = link.select('font[2]/text()').extract() yield item diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 11b1da9d9..93211e912 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -98,31 +98,31 @@ So, by looking at the :ref:`HTML code ` of that page let's construct an XPath (using an HTML selector) for selecting the text inside the title tag:: - >>> hxs.x('//title/text()') + >>> hxs.select('//title/text()') [] -As you can see, the x() method returns a XPathSelectorList, which is a list of +As you can see, the select() method returns a XPathSelectorList, which is a list of new selectors. This API can be used quickly for extracting nested data. To actually extract the textual data you must call the selector ``extract()`` method, as follows:: - >>> hxs.x('//title/text()').extract() + >>> hxs.select('//title/text()').extract() [u'Example website'] Now we're going to get the base URL and some image links:: - >>> hxs.x('//base/@href').extract() + >>> hxs.select('//base/@href').extract() [u'http://example.com/'] - >>> hxs.x('//a[contains(@href, "image")]/@href').extract() + >>> hxs.select('//a[contains(@href, "image")]/@href').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> hxs.x('//a[contains(@href, "image")]/img/@src').extract() + >>> hxs.select('//a[contains(@href, "image")]/img/@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -134,14 +134,14 @@ Using selectors with regular expressions ---------------------------------------- Selectors also have a ``re()`` method for extracting data using regular -expressions. However, unlike using the ``x()`` method, the ``re()`` method does -not return a list of :class:`~scrapy.xpath.XPathSelector` objects, so you can't -construct nested ``.re()`` calls. +expressions. However, unlike using the ``select()`` method, the ``re()`` method +does not return a list of :class:`~scrapy.xpath.XPathSelector` objects, so you +can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code ` above:: - >>> hxs.x('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + >>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') [u'My image 1', u'My image 2', u'My image 3', @@ -153,10 +153,10 @@ Here's an example used to extract images names from the :ref:`HTML code Nesting selectors ----------------- -The ``x()`` selector method returns a list of selectors, so you can call the -``x()`` for those selectors too. Here's an example:: +The ``select()`` selector method returns a list of selectors, so you can call the +``select()`` for those selectors too. Here's an example:: - >>> links = hxs.x('//a[contains(@href, "image")]') + >>> links = hxs.select('//a[contains(@href, "image")]') >>> links.extract() [u'Name: My image 1
      ', u'Name: My image 2
      ', @@ -165,7 +165,7 @@ The ``x()`` selector method returns a list of selectors, so you can call the u'Name: My image 5
      '] >>> for index, link in enumerate(links): - args = (index, link.x('@href').extract(), link.x('img/@src').extract()) + args = (index, link.select('@href').extract(), link.select('img/@src').extract()) print 'Link number %d points to url %s and image %s' % args Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] @@ -186,23 +186,23 @@ to the ``XPathSelector`` you're calling it from. For example, suppose you want to extract all ``

      `` elements inside ``

      `` elements. First you get would get all ``
      `` elements:: - >>> divs = hxs.x('//div') + >>> divs = hxs.select('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``

      `` elements from the document, not only those inside ``

      `` elements:: - >>> for p in divs.x('//p') # this is wrong - gets all

      from the whole document + >>> for p in divs.select('//p') # this is wrong - gets all

      from the whole document >>> print p.extract() This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: - >>> for p in divs.x('.//p') # extracts all

      inside + >>> for p in divs.select('.//p') # extracts all

      inside >>> print p.extract() Another common case would be to extract all direct ``

      `` children:: - >>> for p in divs.x('p') + >>> for p in divs.select('p') >>> print p.extract() For more details about relative XPaths see the `Location Paths`_ section in the diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 7f746d540..5badbcd8b 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -137,7 +137,7 @@ shell works. help -> Python's own help system. object? -> Details about 'object'. ?object also works, ?? prints more. - In [1]: hxs.x("//h2/text()").extract()[2] + In [1]: hxs.select("//h2/text()").extract()[2] Out[1]: u'Welcome to Scrapy' In [2]: get http://slashdot.org @@ -156,10 +156,10 @@ shell works. scrapehelp: Prints this help. ------------------------------------------------------------ - In [3]: hxs.x("//h2/text()").extract() + In [3]: hxs.select("//h2/text()").extract() Out[3]: [u'News for nerds, stuff that matters'] - In [3]: hxs.x("//h2/text()").extract() + In [3]: hxs.select("//h2/text()").extract() Out[3]: [u'News for nerds, stuff that matters'] In [4]: request.method = "POST" diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index d28f92d95..2e9414946 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -20,14 +20,14 @@ class GoogleDirectorySpider(CrawlSpider): hxs = HtmlXPathSelector(response) # The path to website links in directory page - links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') for link in links: item = GoogledirItem() - item.name = link.x('a/text()').extract() - item.url = link.x('a/@href').extract() - item.description = link.x('font[2]/text()').extract() + item.name = link.select('a/text()').extract() + item.url = link.select('a/@href').extract() + item.description = link.select('font[2]/text()').extract() yield item SPIDER = GoogleDirectorySpider() diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 463989ffd..da34cd800 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -35,14 +35,15 @@ class HTMLImageLinkExtractor(object): if selector.xmlNode.type == 'element': if selector.xmlNode.name == 'img': - _add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title')) + _add_link(selector.select('@src'), selector.select('@alt') or \ + selector.select('@title')) else: - children = selector.x('child::*') + children = selector.select('child::*') if len(children): for child in children: ret.extend(self.extract_from_selector(child, parent=selector)) elif selector.xmlNode.name == 'a' and not parent: - _add_link(selector.x('@href'), selector.x('@title')) + _add_link(selector.select('@href'), selector.select('@title')) else: _add_link(selector) @@ -50,13 +51,13 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) - base_url = xs.x('//base/@href').extract() + base_url = xs.select('//base/@href').extract() base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) links = [] for location in self.locations: if isinstance(location, basestring): - selectors = xs.x(location) + selectors = xs.select(location) elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)): selectors = [location] if isinstance(location, HtmlXPathSelector) else location else: diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 67632a7af..710684e72 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -93,7 +93,8 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): if self.restrict_xpaths: hxs = HtmlXPathSelector(response) - html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.x(xpath_expr).extract()) for xpath_expr in self.restrict_xpaths) + html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ + for xpath_expr in self.restrict_xpaths) links = self._extract_links(html_slice, response.url, response.encoding) else: links = BaseSgmlLinkExtractor.extract_links(self, response) diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 587e76cf0..45a1e7bcf 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -88,6 +88,6 @@ class XPathItemLoader(ItemLoader): self.replace_value(field_name, self._get_values(field_name, xpath, re)) def _get_values(self, field_name, xpath, re): - x = self.selector.x(xpath) + x = self.selector.select(xpath) return x.re(re) if re else x.extract() diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 87323efcb..142e8d916 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -74,11 +74,11 @@ class XMLFeedSpider(InitSpider): elif self.iterator == 'xml': selector = XmlXPathSelector(response) self._register_namespaces(selector) - nodes = selector.x('//%s' % self.itertag) + nodes = selector.select('//%s' % self.itertag) elif self.iterator == 'html': selector = HtmlXPathSelector(response) self._register_namespaces(selector) - nodes = selector.x('//%s' % self.itertag) + nodes = selector.select('//%s' % self.itertag) else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index a4d655c26..e534a393f 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -26,7 +26,7 @@ class UtilsIteratorsTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in xmliter(response, 'product'): - attrs.append((x.x("@id").extract(), x.x("name/text()").extract(), x.x("./type/text()").extract())) + attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract())) self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) @@ -34,7 +34,7 @@ class UtilsIteratorsTestCase(unittest.TestCase): def test_xmliter_text(self): body = u"""onetwo""" - self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')], + self.assertEqual([x.select("text()").extract() for x in xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -61,15 +61,15 @@ class UtilsIteratorsTestCase(unittest.TestCase): node = my_iter.next() node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.x('title/text()').extract(), ['Item 1']) - self.assertEqual(node.x('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.x('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.x('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.x('g:price/text()').extract(), ['400']) - self.assertEqual(node.x('image_link/text()').extract(), []) - self.assertEqual(node.x('id/text()').extract(), []) - self.assertEqual(node.x('price/text()').extract(), []) + self.assertEqual(node.select('title/text()').extract(), ['Item 1']) + self.assertEqual(node.select('description/text()').extract(), ['This is item 1']) + self.assertEqual(node.select('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.select('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.select('g:id/text()').extract(), ['ITEM_1']) + self.assertEqual(node.select('g:price/text()').extract(), ['400']) + self.assertEqual(node.select('image_link/text()').extract(), []) + self.assertEqual(node.select('id/text()').extract(), []) + self.assertEqual(node.select('price/text()').extract(), []) def test_xmliter_exception(self): body = u"""onetwo""" diff --git a/scrapy/tests/test_xpath.py b/scrapy/tests/test_xpath.py index a829e388d..08032d9b3 100644 --- a/scrapy/tests/test_xpath.py +++ b/scrapy/tests/test_xpath.py @@ -17,31 +17,31 @@ class XPathSelectorTestCase(unittest.TestCase): response = TextResponse(url="http://example.com", body=body) xpath = HtmlXPathSelector(response) - xl = xpath.x('//input') + xl = xpath.select('//input') self.assertEqual(2, len(xl)) for x in xl: assert isinstance(x, HtmlXPathSelector) - self.assertEqual(xpath.x('//input').extract(), - [x.extract() for x in xpath.x('//input')]) + self.assertEqual(xpath.select('//input').extract(), + [x.extract() for x in xpath.select('//input')]) - self.assertEqual([x.extract() for x in xpath.x("//input[@name='a']/@name")], + self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in xpath.x("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(xpath.x("concat('xpath', 'rules')").extract(), + self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.x("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) @libxml2debug def test_selector_same_type(self): """Test XPathSelector returning the same type in x() method""" text = '

      test

      ' - assert isinstance(XmlXPathSelector(text=text).x("//p")[0], + assert isinstance(XmlXPathSelector(text=text).select("//p")[0], XmlXPathSelector) - assert isinstance(HtmlXPathSelector(text=text).x("//p")[0], + assert isinstance(HtmlXPathSelector(text=text).select("//p")[0], HtmlXPathSelector) @libxml2debug @@ -51,10 +51,10 @@ class XPathSelectorTestCase(unittest.TestCase): # some text which is parsed differently by XML and HTML flavors text = '

      Hello

      ' - self.assertEqual(XmlXPathSelector(text=text).x("//div").extract(), + self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(), [u'

      Hello

      ']) - self.assertEqual(HtmlXPathSelector(text=text).x("//div").extract(), + self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(), [u'

      Hello

      ']) @libxml2debug @@ -76,14 +76,14 @@ class XPathSelectorTestCase(unittest.TestCase): response = HtmlResponse(url="http://example.com", body=body) x = HtmlXPathSelector(response) - divtwo = x.x('//div[@class="two"]') - self.assertEqual(divtwo.x("//li").extract(), + divtwo = x.select('//div[@class="two"]') + self.assertEqual(divtwo.select("//li").extract(), ["
    • one
      • ", "
    • two
      • ", "
    • four
      • ", "
    • five
      • ", "
    • six
      • "]) - self.assertEqual(divtwo.x("./ul/li").extract(), + self.assertEqual(divtwo.select("./ul/li").extract(), ["
    • four
      • ", "
    • five
      • ", "
    • six
      • "]) - self.assertEqual(divtwo.x(".//li").extract(), + self.assertEqual(divtwo.select(".//li").extract(), ["
    • four
      • ", "
    • five
      • ", "
    • six
      • "]) - self.assertEqual(divtwo.x("./li").extract(), + self.assertEqual(divtwo.select("./li").extract(), []) @libxml2debug @@ -103,9 +103,9 @@ class XPathSelectorTestCase(unittest.TestCase): x = HtmlXPathSelector(response) name_re = re.compile("Name: (\w+)") - self.assertEqual(x.x("//ul/li").re(name_re), + self.assertEqual(x.select("//ul/li").re(name_re), ["John", "Paul"]) - self.assertEqual(x.x("//ul/li").re("Age: (\d+)"), + self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), ["10", "20"]) @libxml2debug @@ -119,7 +119,7 @@ class XPathSelectorTestCase(unittest.TestCase): u'lala') xxs = XmlXPathSelector(text='lala') - self.assertEqual(xxs.x('.').extract(), + self.assertEqual(xxs.select('.').extract(), [u'lala']) @@ -136,7 +136,7 @@ class XPathSelectorTestCase(unittest.TestCase): x = XmlXPathSelector(response) x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.x("//somens:a").extract(), + self.assertEqual(x.select("//somens:a").extract(), ['']) @@ -157,12 +157,12 @@ class XPathSelectorTestCase(unittest.TestCase): x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") x.register_namespace("p", "http://www.scrapy.org/product") x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.x("//xmlns:TestTag")), 1) - self.assertEqual(x.x("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.x("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.x("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.x("//p:SecondTestTag").x("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.x("//p:SecondTestTag/xmlns:material").extract()[0], '') + self.assertEqual(len(x.select("//xmlns:TestTag")), 1) + self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:material").extract()[0], '') @libxml2debug def test_selector_invalid_xpath(self): @@ -170,7 +170,7 @@ class XPathSelectorTestCase(unittest.TestCase): x = HtmlXPathSelector(response) xpath = "//test[@foo='bar]" try: - x.x(xpath) + x.select(xpath) except ValueError, e: assert xpath in str(e), "Exception message does not contain invalid xpath" except Exception: @@ -195,7 +195,7 @@ class XPathSelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = HtmlXPathSelector(response) - self.assertEquals(x.x("//span[@id='blank']/text()").extract(), + self.assertEquals(x.select("//span[@id='blank']/text()").extract(), [u'\xa3']) @libxml2debug @@ -223,13 +223,13 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(xxs.extract_unquoted(), u'') - self.assertEqual(xxs.x('/root').extract_unquoted(), [u'']) - self.assertEqual(xxs.x('/root/text()').extract_unquoted(), [ + self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) + self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ u'\n lala\n ', u'\n pff\n']) - self.assertEqual(xxs.x('//*').extract_unquoted(), [u'', u'', u'']) - self.assertEqual(xxs.x('//text()').extract_unquoted(), [ + self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) + self.assertEqual(xxs.select('//text()').extract_unquoted(), [ u'\n lala\n ', u'\n blabla&more', u'a', diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py new file mode 100644 index 000000000..74f393377 --- /dev/null +++ b/scrapy/utils/decorator.py @@ -0,0 +1,24 @@ +import warnings + + +def deprecated(use_instead=None): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used.""" + + def wraps(func): + def new_func(*args, **kwargs): + message = "Call to deprecated function %s." % func.__name__ + if use_instead: + message += " Use %s instead." % use_instead + + warnings.warn_explicit( + message, + category=DeprecationWarning, + filename=func.func_code.co_filename, + lineno=func.func_code.co_firstlineno + 1 + ) + return func(*args, **kwargs) + return new_func + return wraps + diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 6aeb4d675..429d920b6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -27,7 +27,7 @@ def xmliter(obj, nodename): r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end - yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0] + yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] def csviter(obj, delimiter=None, headers=None, encoding=None): """ Returns an iterator of dictionaries from the given csv object diff --git a/scrapy/xpath/selector.py b/scrapy/xpath/selector.py index d91e9a8b6..e89f75c23 100644 --- a/scrapy/xpath/selector.py +++ b/scrapy/xpath/selector.py @@ -11,6 +11,7 @@ from scrapy.xpath.factories import xmlDoc_from_html, xmlDoc_from_xml from scrapy.xpath.document import Libxml2Document from scrapy.utils.python import flatten, unicode_to_str from scrapy.utils.misc import extract_regex +from scrapy.utils.decorator import deprecated class XPathSelector(object): @@ -29,7 +30,7 @@ class XPathSelector(object): self.expr = expr self.response = response - def x(self, xpath): + def select(self, xpath): """Perform the given XPath query on the current XPathSelector and return a XPathSelectorList of the result""" if hasattr(self.xmlNode, 'xpathEval'): @@ -47,7 +48,14 @@ class XPathSelector(object): expr=xpath, response=self.response)]) else: return XPathSelectorList([]) - __call__ = x + + @deprecated(use_instead='XPathSelector.select') + def __call__(self, xpath): + return self.select(xpath) + + @deprecated(use_instead='XPathSelector.select') + def x(self, xpath): + return self.select(xpath) def re(self, regex): """Return a list of unicode strings by applying the regex over all @@ -77,7 +85,7 @@ class XPathSelector(object): def extract_unquoted(self): """Get unescaped contents from the text node (no entities, no CDATA)""" - if self.x('self::text()'): + if self.select('self::text()'): return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') else: return u'' @@ -106,10 +114,14 @@ class XPathSelectorList(list): def __getslice__(self, i, j): return XPathSelectorList(list.__getslice__(self, i, j)) - def x(self, xpath): + def select(self, xpath): """Perform the given XPath query on each XPathSelector of the list and return a new (flattened) XPathSelectorList of the results""" - return XPathSelectorList(flatten([x.x(xpath) for x in self])) + return XPathSelectorList(flatten([x.select(xpath) for x in self])) + + @deprecated(use_instead='XPathSelectorList.select') + def x(self, xpath): + return self.select(xpath) def re(self, regex): """Perform the re() method on each XPathSelector of the list, and