From 6d598f0d94a7b8e8f121e1149c91151ef7c47060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 24 Sep 2013 12:34:46 -0300 Subject: [PATCH] Update selectors docs --- docs/topics/selectors.rst | 283 ++++++++++++++------------------------ scrapy/selector/list.py | 1 + 2 files changed, 108 insertions(+), 176 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index f3a3d5a0d..cd4d825d9 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -25,8 +25,8 @@ either by `XPath`_ or `CSS`_ expressions. used with HTML. `CSS`_ is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements. -Both `lxml`_ and Scrapy Selectors are built over the `libxml2`_ library, which -means they're very similar in speed and parsing accuracy. +Scrapy selectors are built over the `lxml`_ library, which means they're very +similar in speed and parsing accuracy. This page explains how selectors work and describes their API which is very small and simple, unlike the `lxml`_ API which is much bigger because the @@ -41,6 +41,7 @@ reference ` and :ref:`CSS selector reference .. _lxml: http://codespeak.net/lxml/ .. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html .. _libxml2: http://xmlsoft.org/ +.. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors @@ -72,8 +73,8 @@ object as their first parameter. This is the Response they're going to be Example:: - hcs = HtmlCSSSelector(response) # an HTML CSS selector - xxs = XmlXPathSelector(response) # an XML XPath selector + hcs = HtmlCSSSelector(response) # an HTML CSS selector + xxs = XmlXPathSelector(response) # an XML XPath selector Using selectors --------------- @@ -104,9 +105,6 @@ Since we're dealing with HTML, we can use either the :class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, in the ``hxs`` shell variable, or the equivalent :class:`~scrapy.selector.HtmlCSSSelector` found in the ``hcs`` shell variable. -Note that CSS selectors can only select element nodes, while XPath selectors -can select any nodes, including text and comment nodes. There are some methods -to augment CSS selectors with XPath as we'll see below. .. highlight:: python @@ -117,8 +115,9 @@ inside the title tag:: >>> hxs.select('//title/text()') [] -As you can see, the select() method returns an XPathSelectorList, which is a -list of new selectors. This API can be used quickly for extracting nested data. +As you can see, the ``select()`` method returns an +:class:`~scrapy.selector.SelectorList`, which is a list of new selectors. This +API can be used quickly for extracting nested data. To actually extract the textual data, you must call the selector ``extract()`` method, as follows:: @@ -126,12 +125,12 @@ method, as follows:: >>> hxs.select('//title/text()').extract() [u'Example website'] -Now notice that CSS selectors can't select the text nodes. There are some -methods that allow enhancing CSS selectors, such as ``text`` and ``get``:: +Now notice that CSS selectors can select text or attribute nodes using CSS3 +pseudo-elements:: - >>> hcs.select('title').text() + >>> hcs.select('title::text') [] - >>> hcs.select('title').text().extract() + >>> hcs.select('title::text').extract() [u'Example website'] Now we're going to get the base URL and some image links:: @@ -139,7 +138,7 @@ Now we're going to get the base URL and some image links:: >>> hxs.select('//base/@href').extract() [u'http://example.com/'] - >>> hcs.select('base').get('href') + >>> hcs.select('base::attr(href)').extract() [u'http://example.com/'] >>> hxs.select('//a[contains(@href, "image")]/@href').extract() @@ -149,7 +148,7 @@ Now we're going to get the base URL and some image links:: u'image4.html', u'image5.html'] - >>> hcs.select('a[href*=image]').get('href').extract() + >>> hcs.select('a[href*=image]::attr(href)').extract() [u'image1.html', u'image2.html', u'image3.html', @@ -163,7 +162,7 @@ Now we're going to get the base URL and some image links:: u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> hcs.select('a[href*=image] img').get('src').extract() + >>> hcs.select('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -197,28 +196,13 @@ Here's an example:: Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] -The CSSSelectorList ``select`` method will accept CSS selectors, as expected, -but it also provides an ``xpath`` method that accepts XPath selectors to -augment the CSS selectors. Here's an example:: - - >>> links = hcs.select('a[href*=image]') - >>> for index, link in enumerate(links): - args = (index, link.get('href').extract(), link.xpath('img/@src').extract()) - print 'Link number %d points to url %s and image %s' % args - - Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] - Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg'] - Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg'] - Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] - Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] - Using selectors with regular expressions ---------------------------------------- Selectors (both CSS and XPath) also have a ``re()`` method for extracting data using regular expressions. However, unlike using the ``select()`` method, the ``re()`` method does not return a list of -:class:`~scrapy.selector.XPathSelector` objects, so you can't construct nested +:class:`Selector` objects, so you can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code @@ -271,16 +255,71 @@ XPath specification. .. _topics-selectors-ref: Built-in Selectors reference -================================== +============================ .. module:: scrapy.selector :synopsis: Selectors classes There are four types of selectors bundled with Scrapy: :class:`HtmlXPathSelector` and :class:`XmlXPathSelector`, -:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. All of them implement the -same :class:`XPathSelector` interface. The only differences are the selector -syntax and whether it is used to process HTML data or XML data. +:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. + +All of them implement the same :class:`XPathSelector` interface. The only +differences are the selector syntax and whether it is used to process HTML data +or XML data. + +Selector interface +------------------ + +.. class:: Selector(response) + + An instance implementing :class:`Selector` interface is a wrapper over + ``response`` to select certain parts of its content. + + ``response`` is a :class:`~scrapy.http.Response` object that will be used + for selecting and extracting data. + + .. method:: select(query) + + Find nodes matching the selection query and return the result as a + :class:`SelectorList` instance with all elements flattened. List + elements must implement :class:`Selector` interface too. + + .. method:: extract() + + Serialize and return the matched nodes as a list of unicode strings + + .. method:: __nonzero__() + + Returns ``True`` if there is any real content selected or ``False`` + otherwise. In other words, the boolean value of a :class:`Selector` is + given by the contents it selects. + + +SelectorList objects +-------------------- + +.. class:: SelectorList + + The :class:`SelectorList` class is subclass of the builtin ``list`` + class, which provides a few additional methods. + + .. method:: select(query) + + Call the ``select()`` method for each element in this list and return + their results flattened as another :class:`SelectorList`. + + ``query`` is the same argument as the one in :meth:`Selector.select` + + .. method:: extract() + + Call the ``extract()`` method for each element is this list and return + their results flattened, as a list of unicode strings. + + .. method:: __nonzero__() + + returns True if the list is not empty, False otherwise. + .. _topics-xpath-selectors-ref: @@ -289,16 +328,18 @@ XPathSelector objects .. class:: XPathSelector(response) - A :class:`XPathSelector` object is a wrapper over response to select - certain parts of its content. + :class:`Selector` interface implementation that uses `XPath`_ query language + to select content on ``response`` ``response`` is a :class:`~scrapy.http.Response` object that will be used for selecting and extracting data. + In the background, XPath selectors are powered by `lxml`_ library + .. method:: select(xpath) Apply the given XPath relative to this XPathSelector and return a list - of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) + of :class:`XPathSelector` objects (ie. a :class:`SelectorList`) with the result. ``xpath`` is a string containing the XPath to apply @@ -327,50 +368,6 @@ XPathSelector objects Remove all namespaces, allowing to traverse the document using namespace-less xpaths. See example below. - .. method:: __nonzero__() - - Returns ``True`` if there is any real content selected by this - :class:`XPathSelector` or ``False`` otherwise. In other words, the - boolean value of an XPathSelector is given by the contents it selects. - -XPathSelectorList objects -------------------------- - -.. class:: XPathSelectorList - - The :class:`XPathSelectorList` class is subclass of the builtin ``list`` - class, which provides a few additional methods. - - .. method:: select(xpath) - - Call the :meth:`XPathSelector.select` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a new :class:`XPathSelectorList`. - - ``xpath`` is the same argument as the one in - :meth:`XPathSelector.select` - - .. method:: re(regex) - - Call the :meth:`XPathSelector.re` method for all :class:`XPathSelector` - objects in this list and return their results flattened, as a list of - unicode strings. - - ``regex`` is the same argument as the one in :meth:`XPathSelector.re` - - .. method:: extract() - - Call the :meth:`XPathSelector.extract` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a list of unicode strings. - - .. method:: extract_unquoted() - - Call the :meth:`XPathSelector.extract_unoquoted` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a list of unicode strings. This method should not be - applied to all kinds of XPathSelectors. For more info see - :meth:`XPathSelector.extract_unoquoted`. HtmlXPathSelector objects ------------------------- @@ -378,11 +375,9 @@ HtmlXPathSelector objects .. class:: HtmlXPathSelector(response) A subclass of :class:`XPathSelector` for working with HTML content. It uses - the `libxml2`_ HTML parser. See the :class:`XPathSelector` API for more + the `lxml`_ HTML parser. See the :class:`XPathSelector` API for more info. -.. _libxml2: http://xmlsoft.org/ - HtmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -424,7 +419,7 @@ XmlXPathSelector objects .. class:: XmlXPathSelector(response) A subclass of :class:`XPathSelector` for working with XML content. It uses - the `libxml2`_ XML parser. See the :class:`XPathSelector` API for more info. + the `lxml`_ XML parser. See the :class:`XPathSelector` API for more info. XmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -482,11 +477,11 @@ If you wonder why the namespace removal procedure is not always called, instead of having to call it manually. This is because of two reasons which, in order of relevance, are: -1. removing namespaces requires to iterate and modify all nodes in the +1. Removing namespaces requires to iterate and modify all nodes in the document, which is a reasonably expensive operation to performs for all documents crawled by Scrapy -2. there could be some cases where using namespaces is actually required, in +2. There could be some cases where using namespaces is actually required, in case some element names clash between namespaces. These cases are very rare though. @@ -495,144 +490,80 @@ of relevance, are: .. _topics-css-selectors-ref: CSSSelector objects --------------------- +------------------- -.. class:: CSSSelectorMixin(object) +.. class:: CSSSelector(response) - A :class:`CSSSelectorMixin` object is a mixin for either - :class:`XmlXPathSelector` or :class:`HtmlXPathSelector` to select element - nodes using CSS selectors syntax. As a mixin, it is not meant to be used on - its own, but as a secondary parent class. See :class:`XmlCSSSelector` and - :class:`HtmlCSSSelector` for implementations. + :class:`Selector` interface implementation that uses `CSS`_ query language + to select content on ``response`` + + ``response`` is a :class:`~scrapy.http.Response` object that will be used + for selecting and extracting data. + + In the background, CSS selectors are translated into XPath selectors using + `cssselect`_ library and run using :class:`XPathSelector` .. method:: select(css) - Apply the given CSS selector relative to this CSSSelectorMixin and - return a list of :class:`CSSSelectorMixin` objects (ie. a - :class:`CSSSelectorList`) with the result. + Apply the given CSS selector relative to this CSSSelector and return a + :class:`SelectorList` instance. ``css`` is a string containing the CSS selector to apply. - .. method:: xpath(xpath) - - Apply the given XPath relative to this CSSSelectorMixin and return a list - of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) - with the result. - - ``xpath`` is a string containing the XPath to apply. - - .. method:: get(attr) - - Get the attribute relative to this CSSSelectorMixin and return a list - of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) - with the result (usually with one element only). - - ``attr`` is a string containing the attribute name to get. - - .. method:: text(all=False) - - Get the children text nodes relative to this CSSSelectorMixin or, if - ``all`` is True, a string node concatenating all of the descendant text - nodes relative to this CSSSelectorMixin, and return a list of - :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) with - the result. - - ``all`` is a boolean to either select children text nodes (False) or - select a string node concatenating all of the descendant text nodes. - -CSSSelectorList objects ------------------------ - -.. class:: CSSSelectorList - - The :class:`CSSSelectorList` class is subclass of :class:`XPathSelectorList` - which overrides and adds methods to match those of - :class:`CSSSelectorMixin`. - - .. method:: xpath(xpath) - - Call the :meth:`CSSSelectorMixin.xpath` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``xpath`` is the same argument as the one in - :meth:`CSSSelectorMixin.xpath` - - .. method:: get(attr) - - Call the :meth:`CSSSelectorMixin.get` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``attr`` is the same argument as the one in :meth:`CSSSelectorMixin.get` - - .. method:: text(all=False) - - Call the :meth:`CSSSelectorMixin.text` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``all`` is the same argument as the one in :meth:`CSSSelectorMixin.text` - HtmlCSSSelector objects ----------------------- .. class:: HtmlCSSSelector(response) - A subclass of :class:`CSSSelectorMixin` and :class:`HtmlXPathSelector` for - working with HTML content using CSS selectors. + A specialized class for working with HTML content using `CSS`_ selectors. HtmlCSSSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`HtmlCSSSelector` examples to illustrate several concepts. In all cases, we assume there is already an :class:`HtmlCSSSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: +instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: x = HtmlCSSSelector(html_response) 1. Select all ``

`` elements from a HTML response body, returning a list of - :class:`HtmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + :class:`HtmlCSSSelector` objects:: x.select("h1") 2. Extract the text of all ``

`` elements from a HTML response body, returning a list of unicode strings:: - x.select("h1").extract() # this includes the h1 tag - x.select("h1").text().extract() # this excludes the h1 tag + x.select("h1").extract() # Includes the h1 tag + x.select("h1::text").extract() # Only text inside the h1 tag 3. Iterate over all ``

`` tags and print their class attribute:: for node in x.select("p"): - ... print node.get("class").extract() + ... print node.select("::attr(class)").extract() XmlCSSSelector objects ---------------------- .. class:: XmlCSSSelector(response) - A subclass of :class:`CSSSelectorMixin` and :class:`XmlXPathSelector` for - working with XML content using CSS selectors. + A specialized class for working with XML content using `CSS`_ selectors. XmlCSSSelector examples ~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`XmlCSSSelector` examples to illustrate several concepts. In both cases we assume there is already an :class:`XmlCSSSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: +instantiated with a :class:`~scrapy.http.XmlResponse` object like this:: x = XmlCSSSelector(xml_response) 1. Select all ```` elements from a XML response body, returning a list - of :class:`XmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + of :class:`XmlCSSSelector` objects:: x.select("product") -2. Extract all prices from a `Google Base XML feed`_ which requires registering - a namespace:: - - x.register_namespace("g", "http://base.google.com/ns/1.0") - x.xpath("//g:price").extract() +Note that querying xml namespaces with CSS selectors doesn't work, if you are +interesting in this feature consider contributing to `cssselect`_ project. .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py index ea634db1f..774be1155 100644 --- a/scrapy/selector/list.py +++ b/scrapy/selector/list.py @@ -16,6 +16,7 @@ class SelectorList(list): def extract(self): return [x.extract() for x in self] + @deprecated(use_instead='SelectorList.extract') def extract_unquoted(self): return [x.extract_unquoted() for x in self]