mirror of https://github.com/scrapy/scrapy.git
renamed x method of selectors to select
This commit is contained in:
Ismael Carnales
parent
59e0a83ad4
commit
48b40bd620
|
|
@ -132,9 +132,9 @@ Finally, here's the spider code::
|
|||
|
||||
torrent = ScrapedItem()
|
||||
torrent.url = response.url
|
||||
torrent.name = x.x("//h1/text()").extract()
|
||||
torrent.description = x.x("//div[@id='description']").extract()
|
||||
torrent.size = x.x("//div[@id='info-left']/p[2]/text()[2]").extract()
|
||||
torrent.name = x.select("//h1/text()").extract()
|
||||
torrent.description = x.select("//div[@id='description']").extract()
|
||||
torrent.size = x.select("//div[@id='info-left']/p[2]/text()[2]").extract()
|
||||
return [torrent]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -264,19 +264,19 @@ The shell also instantiates two selectors, one for HTML (in the ``hxs``
|
|||
variable) and one for XML (in the ``xxs`` variable)with this response. So let's
|
||||
try them::
|
||||
|
||||
In [1]: hxs.x('/html/head/title')
|
||||
In [1]: hxs.select('/html/head/title')
|
||||
Out[1]: [<HtmlXPathSelector (title) xpath=/html/head/title>]
|
||||
|
||||
In [2]: hxs.x('/html/head/title').extract()
|
||||
In [2]: hxs.select('/html/head/title').extract()
|
||||
Out[2]: [u'<title>Open Directory - Computers: Programming: Languages: Python: Books</title>']
|
||||
|
||||
In [3]: hxs.x('/html/head/title/text()')
|
||||
In [3]: hxs.select('/html/head/title/text()')
|
||||
Out[3]: [<HtmlXPathSelector (text) xpath=/html/head/title/text()>]
|
||||
|
||||
In [4]: hxs.x('/html/head/title/text()').extract()
|
||||
In [4]: hxs.select('/html/head/title/text()').extract()
|
||||
Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books']
|
||||
|
||||
In [5]: hxs.x('/html/head/title/text()').re('(\w+):')
|
||||
In [5]: hxs.select('/html/head/title/text()').re('(\w+):')
|
||||
Out[5]: [u'Computers', u'Programming', u'Languages', u'Python']
|
||||
|
||||
Extracting the data
|
||||
|
|
@ -296,29 +296,29 @@ is inside a ``<ul>`` element, in fact the *second* ``<ul>`` element.
|
|||
So we can select each ``<li>`` element belonging to the sites list with this
|
||||
code::
|
||||
|
||||
hxs.x('//ul[2]/li')
|
||||
hxs.select('//ul[2]/li')
|
||||
|
||||
And from them, the sites descriptions::
|
||||
|
||||
hxs.x('//ul[2]/li/text()').extract()
|
||||
hxs.select('//ul[2]/li/text()').extract()
|
||||
|
||||
The sites titles::
|
||||
|
||||
hxs.x('//ul[2]/li/a/text()').extract()
|
||||
hxs.select('//ul[2]/li/a/text()').extract()
|
||||
|
||||
And the sites links::
|
||||
|
||||
hxs.x('//ul[2]/li/a/@href').extract()
|
||||
hxs.select('//ul[2]/li/a/@href').extract()
|
||||
|
||||
As we said before, each ``x()`` call returns a list of selectors, so we can
|
||||
concatenate further ``x()`` calls to dig deeper into a node. We are going to use
|
||||
As we said before, each ``select()`` call returns a list of selectors, so we can
|
||||
concatenate further ``select()`` calls to dig deeper into a node. We are going to use
|
||||
that property here, so::
|
||||
|
||||
sites = hxs.x('//ul[2]/li')
|
||||
sites = hxs.select('//ul[2]/li')
|
||||
for site in sites:
|
||||
title = site.x('a/text()').extract()
|
||||
link = site.x('a/@href').extract()
|
||||
desc = site.x('text()').extract()
|
||||
title = site.select('a/text()').extract()
|
||||
link = site.select('a/@href').extract()
|
||||
desc = site.select('text()').extract()
|
||||
print title, link, desc
|
||||
|
||||
.. note::
|
||||
|
|
@ -342,11 +342,11 @@ Let's add this code to our spider::
|
|||
|
||||
def parse(self, response):
|
||||
hxs = HtmlXPathSelector(response)
|
||||
sites = hxs.x('//ul[2]/li')
|
||||
sites = hxs.select('//ul[2]/li')
|
||||
for site in sites:
|
||||
title = site.x('a/text()').extract()
|
||||
link = site.x('a/@href').extract()
|
||||
desc = site.x('text()').extract()
|
||||
title = site.select('a/text()').extract()
|
||||
link = site.select('a/@href').extract()
|
||||
desc = site.select('text()').extract()
|
||||
print title, link, desc
|
||||
return []
|
||||
|
||||
|
|
@ -375,13 +375,13 @@ should be like this::
|
|||
|
||||
def parse(self, response):
|
||||
hxs = HtmlXPathSelector(response)
|
||||
sites = hxs.x('//ul[2]/li')
|
||||
sites = hxs.select('//ul[2]/li')
|
||||
items = []
|
||||
for site in sites:
|
||||
item = DmozItem()
|
||||
item.title = site.x('a/text()').extract()
|
||||
item.link = site.x('a/@href').extract()
|
||||
item.desc = site.x('text()').extract()
|
||||
item.title = site.select('a/text()').extract()
|
||||
item.link = site.select('a/@href').extract()
|
||||
item.desc = site.select('text()').extract()
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ XPathSelector objects
|
|||
XPathSelector Methods
|
||||
---------------------
|
||||
|
||||
.. method:: XPathSelector.x(xpath)
|
||||
.. method:: XPathSelector.select(xpath)
|
||||
|
||||
Apply the given XPath relative to this XPathSelector and return a list
|
||||
of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) with
|
||||
|
|
@ -84,7 +84,7 @@ XPathSelectorList objects
|
|||
XPathSelectorList Methods
|
||||
-------------------------
|
||||
|
||||
.. method:: XPathSelectorList.x(xpath)
|
||||
.. method:: XPathSelectorList.select(xpath)
|
||||
|
||||
Call the :meth:`XPathSelector.re` method for all :class:`XPathSelector`
|
||||
objects in this list and return their results flattened, as new
|
||||
|
|
@ -136,27 +136,27 @@ instanced with a :class:`~scrapy.http.Response` object like this::
|
|||
1. Select all ``<h1>`` elements from a HTML response body, returning a list of
|
||||
:class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object)::
|
||||
|
||||
x.x("//h1")
|
||||
x.select("//h1")
|
||||
|
||||
2. Extract the text of all ``<h1>`` elements from a HTML response body,
|
||||
returning a list of unicode strings::
|
||||
|
||||
x.x("//h1").extract() # this includes the h1 tag
|
||||
x.x("//h1/text()").extract() # this excludes the h1 tag
|
||||
x.select("//h1").extract() # this includes the h1 tag
|
||||
x.select("//h1/text()").extract() # this excludes the h1 tag
|
||||
|
||||
3. Iterate over all ``<p>`` tags and print their class attribute::
|
||||
|
||||
for node in x.x("//p"):
|
||||
... print node.x("@href")
|
||||
for node in x.select("//p"):
|
||||
... print node.select("@href")
|
||||
|
||||
4. Extract textual data from all ``<p>`` tags without entities, as a list of
|
||||
unicode strings::
|
||||
|
||||
x.x("//p/text()").extract_unquoted()
|
||||
x.select("//p/text()").extract_unquoted()
|
||||
|
||||
# the following line is wrong. extract_unquoted() should only be used
|
||||
# with textual XPathSelectors
|
||||
x.x("//p").extract_unquoted() # it may work but output is unpredictable
|
||||
x.select("//p").extract_unquoted() # it may work but output is unpredictable
|
||||
|
||||
XmlXPathSelector objects
|
||||
========================
|
||||
|
|
@ -178,12 +178,12 @@ instanced with a :class:`~scrapy.http.Response` object like this::
|
|||
1. Select all ``<product>`` elements from a XML response body, returning a list of
|
||||
:class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object)::
|
||||
|
||||
x.x("//h1")
|
||||
x.select("//h1")
|
||||
|
||||
2. Extract all prices from a `Google Base XML feed`_ which requires registering
|
||||
a namespace::
|
||||
|
||||
x.register_namespace("g", "http://base.google.com/ns/1.0")
|
||||
x.x("//g:price").extract()
|
||||
x.select("//g:price").extract()
|
||||
|
||||
.. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461
|
||||
|
|
|
|||
|
|
@ -189,9 +189,9 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
|
||||
hxs = HtmlXPathSelector(response)
|
||||
item = ScrapedItem()
|
||||
item.id = hxs.x('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
|
||||
item.name = hxs.x('//td[@id="item_name"]/text()').extract()
|
||||
item.description = hxs.x('//td[@id="item_description"]/text()').extract()
|
||||
item.id = hxs.select('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
|
||||
item.name = hxs.select('//td[@id="item_name"]/text()').extract()
|
||||
item.description = hxs.select('//td[@id="item_description"]/text()').extract()
|
||||
return [item]
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
|
@ -307,9 +307,9 @@ These spiders are pretty easy to use, let's have at one example::
|
|||
log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract())))
|
||||
|
||||
item = ScrapedItem()
|
||||
item.id = node.x('@id').extract()
|
||||
item.name = node.x('name').extract()
|
||||
item.description = node.x('description').extract()
|
||||
item.id = node.select('@id').extract()
|
||||
item.name = node.select('name').extract()
|
||||
item.description = node.select('description').extract()
|
||||
return item
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
|
|
|||
|
|
@ -144,13 +144,13 @@ Finally, we can write our ``parse_category()`` method::
|
|||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
# The path to website links in directory page
|
||||
links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
|
||||
links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
|
||||
|
||||
for link in links:
|
||||
item = ScrapedItem()
|
||||
item.name = link.x('a/text()').extract()
|
||||
item.url = link.x('a/@href').extract()
|
||||
item.description = link.x('font[2]/text()').extract()
|
||||
item.name = link.select('a/text()').extract()
|
||||
item.url = link.select('a/@href').extract()
|
||||
item.description = link.select('font[2]/text()').extract()
|
||||
yield item
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -98,31 +98,31 @@ So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that page
|
|||
let's construct an XPath (using an HTML selector) for selecting the text inside
|
||||
the title tag::
|
||||
|
||||
>>> hxs.x('//title/text()')
|
||||
>>> hxs.select('//title/text()')
|
||||
[<HtmlXPathSelector (text) xpath=//title/text()>]
|
||||
|
||||
As you can see, the x() method returns a XPathSelectorList, which is a list of
|
||||
As you can see, the select() method returns a XPathSelectorList, which is a list of
|
||||
new selectors. This API can be used quickly for extracting nested data.
|
||||
|
||||
To actually extract the textual data you must call the selector ``extract()``
|
||||
method, as follows::
|
||||
|
||||
>>> hxs.x('//title/text()').extract()
|
||||
>>> hxs.select('//title/text()').extract()
|
||||
[u'Example website']
|
||||
|
||||
Now we're going to get the base URL and some image links::
|
||||
|
||||
>>> hxs.x('//base/@href').extract()
|
||||
>>> hxs.select('//base/@href').extract()
|
||||
[u'http://example.com/']
|
||||
|
||||
>>> hxs.x('//a[contains(@href, "image")]/@href').extract()
|
||||
>>> hxs.select('//a[contains(@href, "image")]/@href').extract()
|
||||
[u'image1.html',
|
||||
u'image2.html',
|
||||
u'image3.html',
|
||||
u'image4.html',
|
||||
u'image5.html']
|
||||
|
||||
>>> hxs.x('//a[contains(@href, "image")]/img/@src').extract()
|
||||
>>> hxs.select('//a[contains(@href, "image")]/img/@src').extract()
|
||||
[u'image1_thumb.jpg',
|
||||
u'image2_thumb.jpg',
|
||||
u'image3_thumb.jpg',
|
||||
|
|
@ -134,14 +134,14 @@ Using selectors with regular expressions
|
|||
----------------------------------------
|
||||
|
||||
Selectors also have a ``re()`` method for extracting data using regular
|
||||
expressions. However, unlike using the ``x()`` method, the ``re()`` method does
|
||||
not return a list of :class:`~scrapy.xpath.XPathSelector` objects, so you can't
|
||||
construct nested ``.re()`` calls.
|
||||
expressions. However, unlike using the ``select()`` method, the ``re()`` method
|
||||
does not return a list of :class:`~scrapy.xpath.XPathSelector` objects, so you
|
||||
can't construct nested ``.re()`` calls.
|
||||
|
||||
Here's an example used to extract images names from the :ref:`HTML code
|
||||
<topics-selectors-htmlcode>` above::
|
||||
|
||||
>>> hxs.x('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
|
||||
>>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
|
||||
[u'My image 1',
|
||||
u'My image 2',
|
||||
u'My image 3',
|
||||
|
|
@ -153,10 +153,10 @@ Here's an example used to extract images names from the :ref:`HTML code
|
|||
Nesting selectors
|
||||
-----------------
|
||||
|
||||
The ``x()`` selector method returns a list of selectors, so you can call the
|
||||
``x()`` for those selectors too. Here's an example::
|
||||
The ``select()`` selector method returns a list of selectors, so you can call the
|
||||
``select()`` for those selectors too. Here's an example::
|
||||
|
||||
>>> links = hxs.x('//a[contains(@href, "image")]')
|
||||
>>> links = hxs.select('//a[contains(@href, "image")]')
|
||||
>>> links.extract()
|
||||
[u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
|
||||
u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
|
||||
|
|
@ -165,7 +165,7 @@ The ``x()`` selector method returns a list of selectors, so you can call the
|
|||
u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
|
||||
|
||||
>>> for index, link in enumerate(links):
|
||||
args = (index, link.x('@href').extract(), link.x('img/@src').extract())
|
||||
args = (index, link.select('@href').extract(), link.select('img/@src').extract())
|
||||
print 'Link number %d points to url %s and image %s' % args
|
||||
|
||||
Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
|
||||
|
|
@ -186,23 +186,23 @@ to the ``XPathSelector`` you're calling it from.
|
|||
For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
|
||||
elements. First you get would get all ``<div>`` elements::
|
||||
|
||||
>>> divs = hxs.x('//div')
|
||||
>>> divs = hxs.select('//div')
|
||||
|
||||
At first, you may be tempted to use the following approach, which is wrong, as
|
||||
it actually extracts all ``<p>`` elements from the document, not only those
|
||||
inside ``<div>`` elements::
|
||||
|
||||
>>> for p in divs.x('//p') # this is wrong - gets all <p> from the whole document
|
||||
>>> for p in divs.select('//p') # this is wrong - gets all <p> from the whole document
|
||||
>>> print p.extract()
|
||||
|
||||
This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
|
||||
|
||||
>>> for p in divs.x('.//p') # extracts all <p> inside
|
||||
>>> for p in divs.select('.//p') # extracts all <p> inside
|
||||
>>> print p.extract()
|
||||
|
||||
Another common case would be to extract all direct ``<p>`` children::
|
||||
|
||||
>>> for p in divs.x('p')
|
||||
>>> for p in divs.select('p')
|
||||
>>> print p.extract()
|
||||
|
||||
For more details about relative XPaths see the `Location Paths`_ section in the
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ shell works.
|
|||
help -> Python's own help system.
|
||||
object? -> Details about 'object'. ?object also works, ?? prints more.
|
||||
|
||||
In [1]: hxs.x("//h2/text()").extract()[2]
|
||||
In [1]: hxs.select("//h2/text()").extract()[2]
|
||||
Out[1]: u'Welcome to Scrapy'
|
||||
|
||||
In [2]: get http://slashdot.org
|
||||
|
|
@ -156,10 +156,10 @@ shell works.
|
|||
scrapehelp: Prints this help.
|
||||
------------------------------------------------------------
|
||||
|
||||
In [3]: hxs.x("//h2/text()").extract()
|
||||
In [3]: hxs.select("//h2/text()").extract()
|
||||
Out[3]: [u'News for nerds, stuff that matters']
|
||||
|
||||
In [3]: hxs.x("//h2/text()").extract()
|
||||
In [3]: hxs.select("//h2/text()").extract()
|
||||
Out[3]: [u'News for nerds, stuff that matters']
|
||||
|
||||
In [4]: request.method = "POST"
|
||||
|
|
|
|||
|
|
@ -20,14 +20,14 @@ class GoogleDirectorySpider(CrawlSpider):
|
|||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
# The path to website links in directory page
|
||||
links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
|
||||
links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
|
||||
|
||||
for link in links:
|
||||
item = GoogledirItem()
|
||||
|
||||
item.name = link.x('a/text()').extract()
|
||||
item.url = link.x('a/@href').extract()
|
||||
item.description = link.x('font[2]/text()').extract()
|
||||
item.name = link.select('a/text()').extract()
|
||||
item.url = link.select('a/@href').extract()
|
||||
item.description = link.select('font[2]/text()').extract()
|
||||
yield item
|
||||
|
||||
SPIDER = GoogleDirectorySpider()
|
||||
|
|
|
|||
|
|
@ -35,14 +35,15 @@ class HTMLImageLinkExtractor(object):
|
|||
|
||||
if selector.xmlNode.type == 'element':
|
||||
if selector.xmlNode.name == 'img':
|
||||
_add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title'))
|
||||
_add_link(selector.select('@src'), selector.select('@alt') or \
|
||||
selector.select('@title'))
|
||||
else:
|
||||
children = selector.x('child::*')
|
||||
children = selector.select('child::*')
|
||||
if len(children):
|
||||
for child in children:
|
||||
ret.extend(self.extract_from_selector(child, parent=selector))
|
||||
elif selector.xmlNode.name == 'a' and not parent:
|
||||
_add_link(selector.x('@href'), selector.x('@title'))
|
||||
_add_link(selector.select('@href'), selector.select('@title'))
|
||||
else:
|
||||
_add_link(selector)
|
||||
|
||||
|
|
@ -50,13 +51,13 @@ class HTMLImageLinkExtractor(object):
|
|||
|
||||
def extract_links(self, response):
|
||||
xs = HtmlXPathSelector(response)
|
||||
base_url = xs.x('//base/@href').extract()
|
||||
base_url = xs.select('//base/@href').extract()
|
||||
base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url)
|
||||
|
||||
links = []
|
||||
for location in self.locations:
|
||||
if isinstance(location, basestring):
|
||||
selectors = xs.x(location)
|
||||
selectors = xs.select(location)
|
||||
elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)):
|
||||
selectors = [location] if isinstance(location, HtmlXPathSelector) else location
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -93,7 +93,8 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
|
|||
def extract_links(self, response):
|
||||
if self.restrict_xpaths:
|
||||
hxs = HtmlXPathSelector(response)
|
||||
html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.x(xpath_expr).extract()) for xpath_expr in self.restrict_xpaths)
|
||||
html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \
|
||||
for xpath_expr in self.restrict_xpaths)
|
||||
links = self._extract_links(html_slice, response.url, response.encoding)
|
||||
else:
|
||||
links = BaseSgmlLinkExtractor.extract_links(self, response)
|
||||
|
|
|
|||
|
|
@ -88,6 +88,6 @@ class XPathItemLoader(ItemLoader):
|
|||
self.replace_value(field_name, self._get_values(field_name, xpath, re))
|
||||
|
||||
def _get_values(self, field_name, xpath, re):
|
||||
x = self.selector.x(xpath)
|
||||
x = self.selector.select(xpath)
|
||||
return x.re(re) if re else x.extract()
|
||||
|
||||
|
|
|
|||
|
|
@ -74,11 +74,11 @@ class XMLFeedSpider(InitSpider):
|
|||
elif self.iterator == 'xml':
|
||||
selector = XmlXPathSelector(response)
|
||||
self._register_namespaces(selector)
|
||||
nodes = selector.x('//%s' % self.itertag)
|
||||
nodes = selector.select('//%s' % self.itertag)
|
||||
elif self.iterator == 'html':
|
||||
selector = HtmlXPathSelector(response)
|
||||
self._register_namespaces(selector)
|
||||
nodes = selector.x('//%s' % self.itertag)
|
||||
nodes = selector.select('//%s' % self.itertag)
|
||||
else:
|
||||
raise NotSupported('Unsupported node iterator')
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
|
|||
response = XmlResponse(url="http://example.com", body=body)
|
||||
attrs = []
|
||||
for x in xmliter(response, 'product'):
|
||||
attrs.append((x.x("@id").extract(), x.x("name/text()").extract(), x.x("./type/text()").extract()))
|
||||
attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract()))
|
||||
|
||||
self.assertEqual(attrs,
|
||||
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
|
||||
|
|
@ -34,7 +34,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
|
|||
def test_xmliter_text(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
||||
self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')],
|
||||
self.assertEqual([x.select("text()").extract() for x in xmliter(body, 'product')],
|
||||
[[u'one'], [u'two']])
|
||||
|
||||
def test_xmliter_namespaces(self):
|
||||
|
|
@ -61,15 +61,15 @@ class UtilsIteratorsTestCase(unittest.TestCase):
|
|||
|
||||
node = my_iter.next()
|
||||
node.register_namespace('g', 'http://base.google.com/ns/1.0')
|
||||
self.assertEqual(node.x('title/text()').extract(), ['Item 1'])
|
||||
self.assertEqual(node.x('description/text()').extract(), ['This is item 1'])
|
||||
self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
|
||||
self.assertEqual(node.x('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(node.x('g:id/text()').extract(), ['ITEM_1'])
|
||||
self.assertEqual(node.x('g:price/text()').extract(), ['400'])
|
||||
self.assertEqual(node.x('image_link/text()').extract(), [])
|
||||
self.assertEqual(node.x('id/text()').extract(), [])
|
||||
self.assertEqual(node.x('price/text()').extract(), [])
|
||||
self.assertEqual(node.select('title/text()').extract(), ['Item 1'])
|
||||
self.assertEqual(node.select('description/text()').extract(), ['This is item 1'])
|
||||
self.assertEqual(node.select('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
|
||||
self.assertEqual(node.select('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(node.select('g:id/text()').extract(), ['ITEM_1'])
|
||||
self.assertEqual(node.select('g:price/text()').extract(), ['400'])
|
||||
self.assertEqual(node.select('image_link/text()').extract(), [])
|
||||
self.assertEqual(node.select('id/text()').extract(), [])
|
||||
self.assertEqual(node.select('price/text()').extract(), [])
|
||||
|
||||
def test_xmliter_exception(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
|
|
|||
|
|
@ -17,31 +17,31 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
response = TextResponse(url="http://example.com", body=body)
|
||||
xpath = HtmlXPathSelector(response)
|
||||
|
||||
xl = xpath.x('//input')
|
||||
xl = xpath.select('//input')
|
||||
self.assertEqual(2, len(xl))
|
||||
for x in xl:
|
||||
assert isinstance(x, HtmlXPathSelector)
|
||||
|
||||
self.assertEqual(xpath.x('//input').extract(),
|
||||
[x.extract() for x in xpath.x('//input')])
|
||||
self.assertEqual(xpath.select('//input').extract(),
|
||||
[x.extract() for x in xpath.select('//input')])
|
||||
|
||||
self.assertEqual([x.extract() for x in xpath.x("//input[@name='a']/@name")],
|
||||
self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")],
|
||||
[u'a'])
|
||||
self.assertEqual([x.extract() for x in xpath.x("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0'])
|
||||
|
||||
self.assertEqual(xpath.x("concat('xpath', 'rules')").extract(),
|
||||
self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(),
|
||||
[u'xpathrules'])
|
||||
self.assertEqual([x.extract() for x in xpath.x("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_same_type(self):
|
||||
"""Test XPathSelector returning the same type in x() method"""
|
||||
text = '<p>test<p>'
|
||||
assert isinstance(XmlXPathSelector(text=text).x("//p")[0],
|
||||
assert isinstance(XmlXPathSelector(text=text).select("//p")[0],
|
||||
XmlXPathSelector)
|
||||
assert isinstance(HtmlXPathSelector(text=text).x("//p")[0],
|
||||
assert isinstance(HtmlXPathSelector(text=text).select("//p")[0],
|
||||
HtmlXPathSelector)
|
||||
|
||||
@libxml2debug
|
||||
|
|
@ -51,10 +51,10 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
# some text which is parsed differently by XML and HTML flavors
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
|
||||
self.assertEqual(XmlXPathSelector(text=text).x("//div").extract(),
|
||||
self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
self.assertEqual(HtmlXPathSelector(text=text).x("//div").extract(),
|
||||
self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
@libxml2debug
|
||||
|
|
@ -76,14 +76,14 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = HtmlXPathSelector(response)
|
||||
|
||||
divtwo = x.x('//div[@class="two"]')
|
||||
self.assertEqual(divtwo.x("//li").extract(),
|
||||
divtwo = x.select('//div[@class="two"]')
|
||||
self.assertEqual(divtwo.select("//li").extract(),
|
||||
["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.x("./ul/li").extract(),
|
||||
self.assertEqual(divtwo.select("./ul/li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.x(".//li").extract(),
|
||||
self.assertEqual(divtwo.select(".//li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.x("./li").extract(),
|
||||
self.assertEqual(divtwo.select("./li").extract(),
|
||||
[])
|
||||
|
||||
@libxml2debug
|
||||
|
|
@ -103,9 +103,9 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
x = HtmlXPathSelector(response)
|
||||
|
||||
name_re = re.compile("Name: (\w+)")
|
||||
self.assertEqual(x.x("//ul/li").re(name_re),
|
||||
self.assertEqual(x.select("//ul/li").re(name_re),
|
||||
["John", "Paul"])
|
||||
self.assertEqual(x.x("//ul/li").re("Age: (\d+)"),
|
||||
self.assertEqual(x.select("//ul/li").re("Age: (\d+)"),
|
||||
["10", "20"])
|
||||
|
||||
@libxml2debug
|
||||
|
|
@ -119,7 +119,7 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
u'<root>lala</root>')
|
||||
|
||||
xxs = XmlXPathSelector(text='<root>lala</root>')
|
||||
self.assertEqual(xxs.x('.').extract(),
|
||||
self.assertEqual(xxs.select('.').extract(),
|
||||
[u'<root>lala</root>'])
|
||||
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
x = XmlXPathSelector(response)
|
||||
|
||||
x.register_namespace("somens", "http://scrapy.org")
|
||||
self.assertEqual(x.x("//somens:a").extract(),
|
||||
self.assertEqual(x.select("//somens:a").extract(),
|
||||
['<somens:a id="foo"/>'])
|
||||
|
||||
|
||||
|
|
@ -157,12 +157,12 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05")
|
||||
x.register_namespace("p", "http://www.scrapy.org/product")
|
||||
x.register_namespace("b", "http://somens.com")
|
||||
self.assertEqual(len(x.x("//xmlns:TestTag")), 1)
|
||||
self.assertEqual(x.x("//b:Operation/text()").extract()[0], 'hello')
|
||||
self.assertEqual(x.x("//xmlns:TestTag/@b:att").extract()[0], 'value')
|
||||
self.assertEqual(x.x("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
|
||||
self.assertEqual(x.x("//p:SecondTestTag").x("./xmlns:price/text()")[0].extract(), '90')
|
||||
self.assertEqual(x.x("//p:SecondTestTag/xmlns:material").extract()[0], '<material/>')
|
||||
self.assertEqual(len(x.select("//xmlns:TestTag")), 1)
|
||||
self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello')
|
||||
self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value')
|
||||
self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
|
||||
self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90')
|
||||
self.assertEqual(x.select("//p:SecondTestTag/xmlns:material").extract()[0], '<material/>')
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_invalid_xpath(self):
|
||||
|
|
@ -170,7 +170,7 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
x = HtmlXPathSelector(response)
|
||||
xpath = "//test[@foo='bar]"
|
||||
try:
|
||||
x.x(xpath)
|
||||
x.select(xpath)
|
||||
except ValueError, e:
|
||||
assert xpath in str(e), "Exception message does not contain invalid xpath"
|
||||
except Exception:
|
||||
|
|
@ -195,7 +195,7 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = HtmlXPathSelector(response)
|
||||
self.assertEquals(x.x("//span[@id='blank']/text()").extract(),
|
||||
self.assertEquals(x.select("//span[@id='blank']/text()").extract(),
|
||||
[u'\xa3'])
|
||||
|
||||
@libxml2debug
|
||||
|
|
@ -223,13 +223,13 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
|
||||
self.assertEqual(xxs.extract_unquoted(), u'')
|
||||
|
||||
self.assertEqual(xxs.x('/root').extract_unquoted(), [u''])
|
||||
self.assertEqual(xxs.x('/root/text()').extract_unquoted(), [
|
||||
self.assertEqual(xxs.select('/root').extract_unquoted(), [u''])
|
||||
self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [
|
||||
u'\n lala\n ',
|
||||
u'\n pff\n'])
|
||||
|
||||
self.assertEqual(xxs.x('//*').extract_unquoted(), [u'', u'', u''])
|
||||
self.assertEqual(xxs.x('//text()').extract_unquoted(), [
|
||||
self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u''])
|
||||
self.assertEqual(xxs.select('//text()').extract_unquoted(), [
|
||||
u'\n lala\n ',
|
||||
u'\n blabla&more',
|
||||
u'a',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import warnings
|
||||
|
||||
|
||||
def deprecated(use_instead=None):
|
||||
"""This is a decorator which can be used to mark functions
|
||||
as deprecated. It will result in a warning being emitted
|
||||
when the function is used."""
|
||||
|
||||
def wraps(func):
|
||||
def new_func(*args, **kwargs):
|
||||
message = "Call to deprecated function %s." % func.__name__
|
||||
if use_instead:
|
||||
message += " Use %s instead." % use_instead
|
||||
|
||||
warnings.warn_explicit(
|
||||
message,
|
||||
category=DeprecationWarning,
|
||||
filename=func.func_code.co_filename,
|
||||
lineno=func.func_code.co_firstlineno + 1
|
||||
)
|
||||
return func(*args, **kwargs)
|
||||
return new_func
|
||||
return wraps
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ def xmliter(obj, nodename):
|
|||
r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
|
||||
for match in r.finditer(text):
|
||||
nodetext = header_start + match.group() + header_end
|
||||
yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0]
|
||||
yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0]
|
||||
|
||||
def csviter(obj, delimiter=None, headers=None, encoding=None):
|
||||
""" Returns an iterator of dictionaries from the given csv object
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from scrapy.xpath.factories import xmlDoc_from_html, xmlDoc_from_xml
|
|||
from scrapy.xpath.document import Libxml2Document
|
||||
from scrapy.utils.python import flatten, unicode_to_str
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.decorator import deprecated
|
||||
|
||||
class XPathSelector(object):
|
||||
|
||||
|
|
@ -29,7 +30,7 @@ class XPathSelector(object):
|
|||
self.expr = expr
|
||||
self.response = response
|
||||
|
||||
def x(self, xpath):
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on the current XPathSelector and
|
||||
return a XPathSelectorList of the result"""
|
||||
if hasattr(self.xmlNode, 'xpathEval'):
|
||||
|
|
@ -47,7 +48,14 @@ class XPathSelector(object):
|
|||
expr=xpath, response=self.response)])
|
||||
else:
|
||||
return XPathSelectorList([])
|
||||
__call__ = x
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def __call__(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
def re(self, regex):
|
||||
"""Return a list of unicode strings by applying the regex over all
|
||||
|
|
@ -77,7 +85,7 @@ class XPathSelector(object):
|
|||
|
||||
def extract_unquoted(self):
|
||||
"""Get unescaped contents from the text node (no entities, no CDATA)"""
|
||||
if self.x('self::text()'):
|
||||
if self.select('self::text()'):
|
||||
return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
|
||||
else:
|
||||
return u''
|
||||
|
|
@ -106,10 +114,14 @@ class XPathSelectorList(list):
|
|||
def __getslice__(self, i, j):
|
||||
return XPathSelectorList(list.__getslice__(self, i, j))
|
||||
|
||||
def x(self, xpath):
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on each XPathSelector of the list and
|
||||
return a new (flattened) XPathSelectorList of the results"""
|
||||
return XPathSelectorList(flatten([x.x(xpath) for x in self]))
|
||||
return XPathSelectorList(flatten([x.select(xpath) for x in self]))
|
||||
|
||||
@deprecated(use_instead='XPathSelectorList.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
def re(self, regex):
|
||||
"""Perform the re() method on each XPathSelector of the list, and
|
||||
|
|
|
|||
Loading…
Reference in New Issue