From 4645f9e03cf1d0ebe9ad4dd71d1fd6b725c583cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 16:31:20 -0200 Subject: [PATCH] Updates docs to reflect unified selectors api --- docs/intro/overview.rst | 9 +- docs/intro/tutorial.rst | 98 ++++----- docs/topics/firebug.rst | 12 +- docs/topics/leaks.rst | 6 +- docs/topics/loaders.rst | 10 +- docs/topics/selectors.rst | 422 +++++++++++++------------------------- docs/topics/shell.rst | 29 ++- docs/topics/spiders.rst | 47 ++--- sep/sep-020.rst | 8 +- 9 files changed, 254 insertions(+), 387 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index eed4bdac4..2138e76f8 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -143,13 +143,12 @@ Finally, here's the spider code:: rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] def parse_torrent(self, response): - x = HtmlXPathSelector(response) - + ss = Selector(response) torrent = TorrentItem() torrent['url'] = response.url - torrent['name'] = x.select("//h1/text()").extract() - torrent['description'] = x.select("//div[@id='description']").extract() - torrent['size'] = x.select("//div[@id='info-left']/p[2]/text()[2]").extract() + torrent['name'] = ss.xpath("//h1/text()").extract() + torrent['description'] = ss.xpath("//div[@id='description']").extract() + torrent['size'] = ss.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() return torrent For brevity's sake, we intentionally left out the import statements. The diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 495f8b9f8..5068cd8c1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -183,11 +183,12 @@ Introduction to Selectors ^^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to extract data from web pages. Scrapy uses a mechanism -based on `XPath`_ expressions called :ref:`XPath selectors `. -For more information about selectors and other extraction mechanisms see the -:ref:`XPath selectors documentation `. +based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors +`. For more information about selectors and other extraction +mechanisms see the :ref:`Selectors documentation `. .. _XPath: http://www.w3.org/TR/xpath +.. _CSS: http://www.w3.org/TR/selectors Here are some examples of XPath expressions and their meanings: @@ -206,27 +207,28 @@ These are just a couple of simple examples of what you can do with XPath, but XPath expressions are indeed much more powerful. To learn more about XPath we recommend `this XPath tutorial `_. -For working with XPaths, Scrapy provides a :class:`~scrapy.selector.XPathSelector` -class, which comes in two flavours, :class:`~scrapy.selector.HtmlXPathSelector` -(for HTML data) and :class:`~scrapy.selector.XmlXPathSelector` (for XML data). In -order to use them you must instantiate the desired class with a -:class:`~scrapy.http.Response` object. +For working with XPaths, Scrapy provides a :class:`~scrapy.selector.Selector` +class, it must be instantiated with a :class:`~scrapy.http.HtmlResponse` or +:class:`~scrapy.http.XmlResponse` object as first argument. You can see selectors as objects that represent nodes in the document structure. So, the first instantiated selectors are associated to the root node, or the entire document. -Selectors have three methods (click on the method to see the complete API +Selectors have four basic methods (click on the method to see the complete API documentation). -* :meth:`~scrapy.selector.XPathSelector.select`: returns a list of selectors, each of +* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of them representing the nodes selected by the xpath expression given as - argument. + argument. -* :meth:`~scrapy.selector.XPathSelector.extract`: returns a unicode string with - the data selected by the XPath selector. +* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of + them representing the nodes selected by the CSS expression given as argument. -* :meth:`~scrapy.selector.XPathSelector.re`: returns a list of unicode strings +* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the + selected data. + +* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings extracted by applying the regular expression given as argument. @@ -253,12 +255,11 @@ This is what the shell looks like:: [s] Available Scrapy objects: [s] 2010-08-19 21:45:59-0300 [default] INFO: Spider closed (finished) - [s] hxs + [s] ss [s] item Item() [s] request [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Print this help [s] fetch(req_or_url) Fetch a new request or URL and update shell objects @@ -270,23 +271,24 @@ After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body of the response, or you can type ``response.headers`` to see its headers. -The shell also instantiates two selectors, one for HTML (in the ``hxs`` -variable) and one for XML (in the ``xxs`` variable) with this response. So let's -try them:: +The shell also pre-instantiate a selector named ``ss``, it automatically choice +the best parsing rules (XML vs HTML) based on response's type. - In [1]: hxs.select('//title') - Out[1]: [] +So let's try it:: - In [2]: hxs.select('//title').extract() + In [1]: ss.xpath('//title') + Out[1]: [] + + In [2]: ss.xpath('//title').extract() Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: hxs.select('//title/text()') - Out[3]: [] + In [3]: ss.xpath('//title/text()') + Out[3]: [] - In [4]: hxs.select('//title/text()').extract() + In [4]: ss.xpath('//title/text()').extract() Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: hxs.select('//title/text()').re('(\w+):') + In [5]: ss.xpath('//title/text()').re('(\w+):') Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data @@ -306,29 +308,29 @@ is inside a ``
    `` element, in fact the *second* ``
      `` element. So we can select each ``
    • `` element belonging to the sites list with this code:: - hxs.select('//ul/li') + ss.xpath('//ul/li') And from them, the sites descriptions:: - hxs.select('//ul/li/text()').extract() + ss.xpath('//ul/li/text()').extract() The sites titles:: - hxs.select('//ul/li/a/text()').extract() + ss.xpath('//ul/li/a/text()').extract() And the sites links:: - hxs.select('//ul/li/a/@href').extract() + ss.xpath('//ul/li/a/@href').extract() -As we said before, each ``select()`` call returns a list of selectors, so we can -concatenate further ``select()`` calls to dig deeper into a node. We are going to use +As we said before, each ``.xpath()`` call returns a list of selectors, so we can +concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use that property here, so:: - sites = hxs.select('//ul/li') + sites = ss.xpath('//ul/li') for site in sites: - title = site.select('a/text()').extract() - link = site.select('a/@href').extract() - desc = site.select('text()').extract() + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() print title, link, desc .. note:: @@ -341,7 +343,7 @@ that property here, so:: Let's add this code to our spider:: from scrapy.spider import BaseSpider - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector class DmozSpider(BaseSpider): name = "dmoz" @@ -352,12 +354,12 @@ Let's add this code to our spider:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - sites = hxs.select('//ul/li') + ss = Selector(response) + sites = ss.xpath('//ul/li') for site in sites: - title = site.select('a/text()').extract() - link = site.select('a/@href').extract() - desc = site.select('text()').extract() + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() print title, link, desc Now try crawling the dmoz.org domain again and you'll see sites being printed @@ -382,7 +384,7 @@ Spiders are expected to return their scraped data inside scraped so far, the final code for our Spider would be like this:: from scrapy.spider import BaseSpider - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from tutorial.items import DmozItem @@ -395,14 +397,14 @@ scraped so far, the final code for our Spider would be like this:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - sites = hxs.select('//ul/li') + ss = Selector(response) + sites = ss.xpath('//ul/li') items = [] for site in sites: item = DmozItem() - item['title'] = site.select('a/text()').extract() - item['link'] = site.select('a/@href').extract() - item['desc'] = site.select('text()').extract() + item['title'] = site.xpath('a/text()').extract() + item['link'] = site.xpath('a/@href').extract() + item['desc'] = site.xpath('text()').extract() items.append(item) return items diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 395f3bfe1..839701a32 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -107,7 +107,7 @@ Now we're going to write the code to extract data from those pages. With the help of Firebug, we'll take a look at some page containing links to websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can -extract those links using :ref:`XPath selectors `. We'll also +extract those links using :ref:`Selectors `. We'll also use the :ref:`Scrapy shell ` to test those XPath's and make sure they work as we expect. @@ -146,16 +146,16 @@ that have that grey colour of the links, Finally, we can write our ``parse_category()`` method:: def parse_category(self, response): - hxs = HtmlXPathSelector(response) + ss = Selector(response) # The path to website links in directory page - links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + links = ss.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') for link in links: item = DirectoryItem() - item['name'] = link.select('a/text()').extract() - item['url'] = link.select('a/@href').extract() - item['description'] = link.select('font[2]/text()').extract() + item['name'] = link.xpath('a/text()').extract() + item['url'] = link.xpath('a/@href').extract() + item['description'] = link.xpath('font[2]/text()').extract() yield item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 95a41f360..de9bf645e 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -67,7 +67,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:: ExampleSpider 1 oldest: 15s ago HtmlResponse 10 oldest: 1s ago - XPathSelector 2 oldest: 0s ago + Selector 2 oldest: 0s ago FormRequest 878 oldest: 7s ago As you can see, that report also shows the "age" of the oldest object in each @@ -87,7 +87,7 @@ subclasses): * ``scrapy.http.Request`` * ``scrapy.http.Response`` * ``scrapy.item.Item`` -* ``scrapy.selector.XPathSelector`` +* ``scrapy.selector.Selector`` * ``scrapy.spider.BaseSpider`` * ``scrapy.selector.document.Libxml2Document`` @@ -117,7 +117,7 @@ references:: SomenastySpider 1 oldest: 15s ago HtmlResponse 3890 oldest: 265s ago - XPathSelector 2 oldest: 0s ago + Selector 2 oldest: 0s ago Request 3878 oldest: 250s ago The fact that there are so many live responses (and that they're so old) is diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index bab3d6113..47c06896b 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -31,7 +31,7 @@ using the Item class specified in the :attr:`ItemLoader.default_item_class` attribute. Then, you start collecting values into the Item Loader, typically using -:ref:`XPath Selectors `. You can add more than one value to +:ref:`Selectors `. You can add more than one value to the same item field; the Item Loader will know how to "join" those values later using a proper processing function. @@ -352,14 +352,14 @@ ItemLoader objects The :class:`XPathItemLoader` class extends the :class:`ItemLoader` class providing more convenient mechanisms for extracting data from web pages - using :ref:`XPath selectors `. + using :ref:`selectors `. :class:`XPathItemLoader` objects accept two more additional parameters in their constructors: :param selector: The selector to extract data from, when using the :meth:`add_xpath` or :meth:`replace_xpath` method. - :type selector: :class:`~scrapy.selector.XPathSelector` object + :type selector: :class:`~scrapy.selector.Selector` object :param response: The response used to construct the selector using the :attr:`default_selector_class`, unless the selector argument is given, @@ -418,7 +418,7 @@ ItemLoader objects .. attribute:: selector - The :class:`~scrapy.selector.XPathSelector` object to extract data from. + The :class:`~scrapy.selector.Selector` object to extract data from. It's either the selector given in the constructor or one created from the response given in the constructor using the :attr:`default_selector_class`. This attribute is meant to be @@ -592,7 +592,7 @@ Here is a list of all built-in processors: work with single values (instead of iterables). For this reason the :class:`MapCompose` processor is typically used as input processor, since data is often extracted using the - :meth:`~scrapy.selector.XPathSelector.extract` method of :ref:`selectors + :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors `, which returns a list of unicode strings. The example below should clarify how it works:: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 554f16eb6..b9b185d4c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -33,9 +33,8 @@ small and simple, unlike the `lxml`_ API which is much bigger because the `lxml`_ library can be used for many other tasks, besides selecting markup documents. -For a complete reference of the selectors API see :ref:`XPath selector -reference ` and :ref:`CSS selector reference -`. +For a complete reference of the selectors API see +:ref:`Selector reference ` .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://codespeak.net/lxml/ @@ -44,36 +43,33 @@ reference ` and :ref:`CSS selector reference .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors + Using selectors =============== Constructing selectors ---------------------- -There are four types of selectors bundled with Scrapy. Those are: - - * :class:`~scrapy.selector.HtmlXPathSelector` - for working with HTML - documents using XPath. - - * :class:`~scrapy.selector.XmlXPathSelector` - for working with XML documents - using XPath. - - * :class:`~scrapy.selector.HtmlCSSSelector` - for working with HTML documents - using CSS selectors. - - * :class:`~scrapy.selector.XmlCSSSelector` - for working with XML documents - using CSS selectors. - .. highlight:: python -All of them share the same selector API, and are constructed with a Response -object as their first parameter. This is the Response they're going to be -"selecting". +Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +constructed by passing a `Response` object as first argument, the response's +body is what they're going to be "selecting":: -Example:: + from scrapy.spider import BaseSpider + from scrapy.selector import Selector + + class MySpider(BaseSpider): + # ... + def parse(self, response): + ss = Selector(response) + # Using XPath query + print ss.xpath('//p') + # Using CSS query + print ss.css('p') + # Nesting queries + print ss.xpath('//div[@foo="bar"]').css('span#bold') - hcs = HtmlCSSSelector(response) # an HTML CSS selector - xxs = XmlXPathSelector(response) # an XML XPath selector Using selectors --------------- @@ -97,13 +93,10 @@ First, let's open the shell:: scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html -Then, after the shell loads, you'll have some selectors already instantiated -and ready to use. +Then, after the shell loads, you'll have a selector already instantiated and +ready to use in ``ss`` shell variable. -Since we're dealing with HTML, we can use either the -:class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, -in the ``hxs`` shell variable, or the equivalent -:class:`~scrapy.selector.HtmlCSSSelector` found in the ``hcs`` shell variable. +Since we're dealing with HTML, the selector will automatically use an HTML parser. .. highlight:: python @@ -111,57 +104,55 @@ So, by looking at the :ref:`HTML code ` of that page, let's construct an XPath (using an HTML selector) for selecting the text inside the title tag:: - >>> hxs.select('//title/text()') - [] + >>> ss.xpath('//title/text()') + [] -As you can see, the ``select()`` method returns an -:class:`~scrapy.selector.SelectorList`, which is a list of new selectors. This -API can be used quickly for extracting nested data. +As you can see, the ``.xpath()`` method returns an +:class:`~scrapy.selector.SelectorList` instance, which is a list of new +selectors. This API can be used quickly for extracting nested data. -To actually extract the textual data, you must call the selector ``extract()`` +To actually extract the textual data, you must call the selector ``.extract()`` method, as follows:: - >>> hxs.select('//title/text()').extract() + >>> ss.xpath('//title/text()').extract() [u'Example website'] -Now notice that CSS selectors can select text or attribute nodes using CSS3 +Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: - >>> hcs.select('title::text') - [] - >>> hcs.select('title::text').extract() + >>> ss.css('title::text').extract() [u'Example website'] Now we're going to get the base URL and some image links:: - >>> hxs.select('//base/@href').extract() + >>> ss.xpath('//base/@href').extract() [u'http://example.com/'] - >>> hcs.select('base::attr(href)').extract() + >>> ss.css('base::attr(href)').extract() [u'http://example.com/'] - >>> hxs.select('//a[contains(@href, "image")]/@href').extract() + >>> ss.xpath('//a[contains(@href, "image")]/@href').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> hcs.select('a[href*=image]::attr(href)').extract() + >>> ss.css('a[href*=image]::attr(href)').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> hxs.select('//a[contains(@href, "image")]/img/@src').extract() + >>> ss.xpath('//a[contains(@href, "image")]/img/@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> hcs.select('a[href*=image] img::attr(src)').extract() + >>> ss.css('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -173,11 +164,11 @@ Now we're going to get the base URL and some image links:: Nesting selectors ----------------- -The ``select()`` selector method returns a list of selectors of the same type -(XPath or CSS), so you can call the ``select()`` for those selectors too. -Here's an example:: +The selection methods (``.xpath()`` or ``.css()``) returns a list of selectors +of the same type, so you can call the selection methods for those selectors +too. Here's an example:: - >>> links = hxs.select('//a[contains(@href, "image")]') + >>> links = ss.xpath('//a[contains(@href, "image")]') >>> links.extract() [u'Name: My image 1
      ', u'Name: My image 2
      ', @@ -186,7 +177,7 @@ Here's an example:: u'Name: My image 5
      '] >>> for index, link in enumerate(links): - args = (index, link.select('@href').extract(), link.select('img/@src').extract()) + args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract()) print 'Link number %d points to url %s and image %s' % args Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] @@ -198,16 +189,15 @@ Here's an example:: Using selectors with regular expressions ---------------------------------------- -Selectors (both CSS and XPath) also have a ``re()`` method for extracting data -using regular expressions. However, unlike using the ``select()`` method, the -``re()`` method does not return a list of -:class:`Selector` objects, so you can't construct nested -``.re()`` calls. +:class:`~scrapy.selector.Selector` also have a ``.re()`` method for extracting +data using regular expressions. However, unlike using ``.xpath()`` or +``.css()`` methods, ``.re()`` method returns a list of unicode strings. So you +can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code ` above:: - >>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + >>> ss.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') [u'My image 1', u'My image 2', u'My image 3', @@ -219,30 +209,30 @@ Here's an example used to extract images names from the :ref:`HTML code Working with relative XPaths ---------------------------- -Keep in mind that if you are nesting XPathSelectors and use an XPath that -starts with ``/``, that XPath will be absolute to the document and not relative -to the ``XPathSelector`` you're calling it from. +Keep in mind that if you are nesting selectors and use an XPath that starts +with ``/``, that XPath will be absolute to the document and not relative to the +``Selector`` you're calling it from. For example, suppose you want to extract all ``

      `` elements inside ``

      `` elements. First, you would get all ``
      `` elements:: - >>> divs = hxs.select('//div') + >>> divs = ss.xpath('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``

      `` elements from the document, not only those inside ``

      `` elements:: - >>> for p in divs.select('//p') # this is wrong - gets all

      from the whole document + >>> for p in divs.xpath('//p') # this is wrong - gets all

      from the whole document >>> print p.extract() This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: - >>> for p in divs.select('.//p') # extracts all

      inside + >>> for p in divs.xpath('.//p') # extracts all

      inside >>> print p.extract() Another common case would be to extract all direct ``

      `` children:: - >>> for p in divs.select('p') + >>> for p in divs.xpath('p') >>> print p.extract() For more details about relative XPaths see the `Location Paths`_ section in the @@ -257,42 +247,67 @@ Built-in Selectors reference ============================ .. module:: scrapy.selector - :synopsis: Selectors classes + :synopsis: Selector class -There are four types of selectors bundled with Scrapy: -:class:`HtmlXPathSelector` and :class:`XmlXPathSelector`, -:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. +.. class:: Selector(response, contenttype=None) -All of them implement the same :class:`XPathSelector` interface. The only -differences are the selector syntax and whether it is used to process HTML data -or XML data. + An instance of :class:`Selector` is a wrapper over ``response`` to select + certain parts of its content. -Selector interface ------------------- + ``response`` is a :class:`~scrapy.http.HtmlResponse` or + :class:`~scrapy.http.XmlResponse` object that will be used for selecting and + extracting data. -.. class:: Selector(response) + ``contenttype`` tells what parser and selection flavor is used to parse the + response body. It defaults to ``"html"`` for :class:`~scrapy.http.HtmlResponse` + and ``"xml"`` for :class:`~scrapy.http.XmlResponse`. - An instance implementing :class:`Selector` interface is a wrapper over - ``response`` to select certain parts of its content. + .. method:: xpath(query) - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. + Find nodes matching the xpath ``query`` and return the result as a + :class:`SelectorList` instance with all elements flattened. List + elements implement :class:`Selector` interface too. - .. method:: select(query) + ``query`` is a string containing the XPATH query to apply. - Find nodes matching the selection query and return the result as a - :class:`SelectorList` instance with all elements flattened. List - elements must implement :class:`Selector` interface too. + .. method:: css(query) - .. method:: extract() + Apply the given CSS selector and return a :class:`SelectorList` instance. - Serialize and return the matched nodes as a list of unicode strings + ``query`` is a string containing the CSS selector to apply. - .. method:: __nonzero__() + In the background, CSS queries are translated into XPath queries using + `cssselect`_ library and run ``.xpath()`` method. - Returns ``True`` if there is any real content selected or ``False`` - otherwise. In other words, the boolean value of a :class:`Selector` is - given by the contents it selects. + .. method:: extract() + + Serialize and return the matched nodes as a list of unicode strings. + Percent encoded content is unquoted. + + .. method:: re(regex) + + Apply the given regex and return a list of unicode strings with the + matches. + + ``regex`` can be either a compiled regular expression or a string which + will be compiled to a regular expression using ``re.compile(regex)`` + + .. method:: register_namespace(prefix, uri) + + Register the given namespace to be used in this :class:`Selector`. + Without registering namespaces you can't select or extract data from + non-standard namespaces. See examples below. + + .. method:: remove_namespaces() + + Remove all namespaces, allowing to traverse the document using + namespace-less xpaths. See example below. + + .. method:: __nonzero__() + + Returns ``True`` if there is any real content selected or ``False`` + otherwise. In other words, the boolean value of a :class:`Selector` is + given by the contents it selects. SelectorList objects @@ -303,16 +318,28 @@ SelectorList objects The :class:`SelectorList` class is subclass of the builtin ``list`` class, which provides a few additional methods. - .. method:: select(query) + .. method:: xpath(query) - Call the ``select()`` method for each element in this list and return + Call the ``.xpath()`` method for each element in this list and return their results flattened as another :class:`SelectorList`. - ``query`` is the same argument as the one in :meth:`Selector.select` + ``query`` is the same argument as the one in :meth:`Selector.xpath` + + .. method:: css(query) + + Call the ``.css()`` method for each element in this list and return + their results flattened as another :class:`SelectorList`. + + ``query`` is the same argument as the one in :meth:`Selector.css` .. method:: extract() - Call the ``extract()`` method for each element is this list and return + Call the ``.extract()`` method for each element is this list and return + their results flattened, as a list of unicode strings. + + .. method:: re() + + Call the ``.re()`` method for each element is this list and return their results flattened, as a list of unicode strings. .. method:: __nonzero__() @@ -320,126 +347,50 @@ SelectorList objects returns True if the list is not empty, False otherwise. -.. _topics-xpath-selectors-ref: +Selector examples on HTML response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -XPathSelector objects ---------------------- +Here's a couple of :class:`Selector` examples to illustrate several concepts. +In all cases, we assume there is already an :class:`Selector` instantiated with +a :class:`~scrapy.http.HtmlResponse` object like this:: -.. class:: XPathSelector(response) - - :class:`Selector` interface implementation that uses `XPath`_ query language - to select content on ``response`` - - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. - - In the background, XPath selectors are powered by `lxml`_ library - - .. method:: select(xpath) - - Apply the given XPath relative to this XPathSelector and return a list - of :class:`XPathSelector` objects (ie. a :class:`SelectorList`) - with the result. - - ``xpath`` is a string containing the XPath to apply - - .. method:: re(regex) - - Apply the given regex and return a list of unicode strings with the - matches. - - ``regex`` can be either a compiled regular expression or a string which - will be compiled to a regular expression using ``re.compile(regex)`` - - .. method:: extract() - - Return a unicode string with the content of this :class:`XPathSelector` - object. - - .. method:: register_namespace(prefix, uri) - - Register the given namespace to be used in this :class:`XPathSelector`. - Without registering namespaces you can't select or extract data from - non-standard namespaces. See examples below. - - .. method:: remove_namespaces() - - Remove all namespaces, allowing to traverse the document using - namespace-less xpaths. See example below. - - -HtmlXPathSelector objects -------------------------- - -.. class:: HtmlXPathSelector(response) - - A subclass of :class:`XPathSelector` for working with HTML content. It uses - the `lxml`_ HTML parser. See the :class:`XPathSelector` API for more - info. - -HtmlXPathSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`HtmlXPathSelector` examples to illustrate several -concepts. In all cases, we assume there is already an -:class:`HtmlXPathSelector` instantiated with a :class:`~scrapy.http.Response` -object like this:: - - x = HtmlXPathSelector(html_response) + x = Selector(html_response) 1. Select all ``

      `` elements from a HTML response body, returning a list of - :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object):: + :class:`Selector` objects (ie. a :class:`SelectorList` object):: - x.select("//h1") + x.xpath("//h1") 2. Extract the text of all ``

      `` elements from a HTML response body, returning a list of unicode strings:: - x.select("//h1").extract() # this includes the h1 tag - x.select("//h1/text()").extract() # this excludes the h1 tag + x.xpath("//h1").extract() # this includes the h1 tag + x.xpath("//h1/text()").extract() # this excludes the h1 tag 3. Iterate over all ``

      `` tags and print their class attribute:: - for node in x.select("//p"): - ... print node.select("@class").extract() + for node in x.xpath("//p"): + ... print node.xpath("@class").extract() -4. Extract textual data from all ``

      `` tags without entities, as a list of - unicode strings:: +Selector examples on XML response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - x.select("//p/text()").extract_unquoted() +Here's a couple of examples to illustrate several concepts. In both cases we +assume there is already an :class:`Selector` instantiated with a +:class:`~scrapy.http.XmlResponse` object like this:: - # the following line is wrong. extract_unquoted() should only be used - # with textual XPathSelectors - x.select("//p").extract_unquoted() # it may work but output is unpredictable - -XmlXPathSelector objects ------------------------- - -.. class:: XmlXPathSelector(response) - - A subclass of :class:`XPathSelector` for working with XML content. It uses - the `lxml`_ XML parser. See the :class:`XPathSelector` API for more info. - -XmlXPathSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`XmlXPathSelector` examples to illustrate several -concepts. In both cases we assume there is already an :class:`XmlXPathSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: - - x = XmlXPathSelector(xml_response) + x = Selector(xml_response) 1. Select all ```` elements from a XML response body, returning a list - of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` - object):: + of :class:`Selector` objects (ie. a :class:`SelectorList` object):: - x.select("//product") + x.xpath("//product") 2. Extract all prices from a `Google Base XML feed`_ which requires registering a namespace:: x.register_namespace("g", "http://base.google.com/ns/1.0") - x.select("//g:price").extract() + x.xpath("//g:price").extract() .. _removing-namespaces: @@ -449,7 +400,7 @@ Removing namespaces When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the -:meth:`XPathSelector.remove_namespaces` method for that. +:meth:`Selector.remove_namespaces` method for that. Let's show an example that illustrates this with Github blog atom feed. @@ -460,16 +411,16 @@ First, we open the shell with the url we want to scrape:: Once in the shell we can try selecting all ```` objects and see that it doesn't work (because the Atom XML namespace is obfuscating those nodes):: - >>> xxs.select("//link") + >>> xxs.xpath("//link") [] -But once we call the :meth:`XPathSelector.remove_namespaces` method, all +But once we call the :meth:`Selector.remove_namespaces` method, all nodes can be accessed directly by their names:: >>> xxs.remove_namespaces() - >>> xxs.select("//link") - [, + >>> xxs.xpath("//link") + [, ... If you wonder why the namespace removal procedure is not always called, instead @@ -485,84 +436,3 @@ of relevance, are: though. .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 - -.. _topics-css-selectors-ref: - -CSSSelector objects -------------------- - -.. class:: CSSSelector(response) - - :class:`Selector` interface implementation that uses `CSS`_ query language - to select content on ``response`` - - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. - - In the background, CSS selectors are translated into XPath selectors using - `cssselect`_ library and run using :class:`XPathSelector` - - .. method:: select(css) - - Apply the given CSS selector relative to this CSSSelector and return a - :class:`SelectorList` instance. - - ``css`` is a string containing the CSS selector to apply. - -HtmlCSSSelector objects ------------------------ - -.. class:: HtmlCSSSelector(response) - - A specialized class for working with HTML content using `CSS`_ selectors. - -HtmlCSSSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`HtmlCSSSelector` examples to illustrate several -concepts. In all cases, we assume there is already an :class:`HtmlCSSSelector` -instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: - - x = HtmlCSSSelector(html_response) - -1. Select all ``

      `` elements from a HTML response body, returning a list of - :class:`HtmlCSSSelector` objects:: - - x.select("h1") - -2. Extract the text of all ``

      `` elements from a HTML response body, - returning a list of unicode strings:: - - x.select("h1").extract() # Includes the h1 tag - x.select("h1::text").extract() # Only text inside the h1 tag - -3. Iterate over all ``

      `` tags and print their class attribute:: - - for node in x.select("p"): - ... print node.select("::attr(class)").extract() - -XmlCSSSelector objects ----------------------- - -.. class:: XmlCSSSelector(response) - - A specialized class for working with XML content using `CSS`_ selectors. - -XmlCSSSelector examples -~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`XmlCSSSelector` examples to illustrate several -concepts. In both cases we assume there is already an :class:`XmlCSSSelector` -instantiated with a :class:`~scrapy.http.XmlResponse` object like this:: - - x = XmlCSSSelector(xml_response) - -1. Select all ```` elements from a XML response body, returning a list - of :class:`XmlCSSSelector` objects:: - - x.select("product") - -Note that querying xml namespaces with CSS selectors doesn't work, if you are -interesting in this feature consider contributing to `cssselect`_ project. - -.. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index daa8d83d0..589142aa6 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -9,10 +9,10 @@ scraping code very quickly, without having to run the spider. It's meant to be used for testing data extraction code, but you can actually use it for testing any kind of code as it is also a regular Python shell. -The shell is used for testing XPath expressions and see how they work and what -data they extract from the web pages you're trying to scrape. It allows you to -interactively test your XPaths while you're writing your spider, without having -to run the spider to test every change. +The shell is used for testing XPath or CSS expressions and see how they work +and what data they extract from the web pages you're trying to scrape. It +allows you to interactively test your expressions while you're writing your +spider, without having to run the spider to test every change. Once you get familiarized with the Scrapy shell, you'll see that it's an invaluable tool for developing and debugging your spiders. @@ -66,7 +66,7 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.XPathSelector` objects (for both HTML and XML +:class:`~scrapy.selector.Selector` objects (for both HTML and XML content). Those objects are: @@ -83,10 +83,7 @@ Those objects are: * ``response`` - a :class:`~scrapy.http.Response` object containing the last fetched page - * ``hxs`` - a :class:`~scrapy.selector.HtmlXPathSelector` object constructed - with the last response fetched - - * ``xxs`` - a :class:`~scrapy.selector.XmlXPathSelector` object constructed + * ``ss`` - a :class:`~scrapy.selector.Selector` object constructed with the last response fetched * ``settings`` - the current :ref:`Scrapy settings ` @@ -114,13 +111,12 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available objects - [s] hxs + [s] ss [s] item Item() [s] request [s] response [s] settings [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Prints this help. [s] fetch(req_or_url) Fetch a new request or URL and update objects @@ -130,24 +126,23 @@ all start with the ``[s]`` prefix):: After that, we can star playing with the objects:: - >>> hxs.select("//h2/text()").extract()[0] + >>> ss.xpath("//h2/text()").extract()[0] u'Welcome to Scrapy' >>> fetch("http://slashdot.org") [s] Available Scrapy objects: - [s] hxs + [s] ss [s] item JobItem() [s] request [s] response <200 http://slashdot.org> [s] settings [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> hxs.select("//h2/text()").extract() + >>> ss.xpath("//h2/text()").extract() [u'News for nerds, stuff that matters'] >>> request = request.replace(method="POST") @@ -185,7 +180,7 @@ When you run the spider, you will get something similar to this:: 2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled (referer: ) 2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled (referer: ) [s] Available objects - [s] hxs + [s] ss ... >>> response.url @@ -193,7 +188,7 @@ When you run the spider, you will get something similar to this:: Then, you can check if the extraction code is working:: - >>> hxs.select('//h1') + >>> ss.xpath('//h1') [] Nope, it doesn't. So you can open the response in your web browser and see if diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 82e96df1e..28393dfe2 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -216,7 +216,7 @@ Let's see an example:: Another example returning multiples Requests and Items from a single callback:: - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from scrapy.spider import BaseSpider from scrapy.http import Request from myproject.items import MyItem @@ -231,11 +231,11 @@ Another example returning multiples Requests and Items from a single callback:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - for h3 in hxs.select('//h3').extract(): + ss = Selector(response) + for h3 in ss.xpath('//h3').extract(): yield MyItem(title=h3) - for url in hxs.select('//a/@href').extract(): + for url in ss.xpath('//a/@href').extract(): yield Request(url, callback=self.parse) .. module:: scrapy.contrib.spiders @@ -314,7 +314,7 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from scrapy.item import Item class MySpider(CrawlSpider): @@ -334,11 +334,11 @@ Let's now take a look at an example CrawlSpider with rules:: def parse_item(self, response): self.log('Hi, this is an item page! %s' % response.url) - hxs = HtmlXPathSelector(response) + ss = Selector(response) item = Item() - item['id'] = hxs.select('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = hxs.select('//td[@id="item_name"]/text()').extract() - item['description'] = hxs.select('//td[@id="item_description"]/text()').extract() + item['id'] = ss.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') + item['name'] = ss.xpath('//td[@id="item_name"]/text()').extract() + item['description'] = ss.xpath('//td[@id="item_description"]/text()').extract() return item @@ -366,15 +366,15 @@ XMLFeedSpider A string which defines the iterator to use. It can be either: - - ``'iternodes'`` - a fast iterator based on regular expressions + - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses HtmlXPathSelector. Keep in mind - this uses DOM parsing and must load all DOM in memory which could be a - problem for big feeds + - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + Keep in mind this uses DOM parsing and must load all DOM in memory + which could be a problem for big feeds - - ``'xml'`` - an iterator which uses XmlXPathSelector. Keep in mind - this uses DOM parsing and must load all DOM in memory which could be a - problem for big feeds + - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + Keep in mind this uses DOM parsing and must load all DOM in memory + which could be a problem for big feeds It defaults to: ``'iternodes'``. @@ -390,7 +390,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.XPathSelector.register_namespace` method. + :meth:`~scrapy.selector.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -416,9 +416,10 @@ XMLFeedSpider .. method:: parse_node(response, selector) This method is called for the nodes matching the provided tag name - (``itertag``). Receives the response and an XPathSelector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either a :class:`~scrapy.item.Item` object, a + (``itertag``). Receives the response and an + :class:`~scrapy.selector.Selector` for each node. Overriding this + method is mandatory. Otherwise, you spider won't work. This method + must return either a :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request` object, or an iterable containing any of them. @@ -451,9 +452,9 @@ These spiders are pretty easy to use, let's have a look at one example:: log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) item = Item() - item['id'] = node.select('@id').extract() - item['name'] = node.select('name').extract() - item['description'] = node.select('description').extract() + item['id'] = node.xpath('@id').extract() + item['name'] = node.xpath('name').extract() + item['description'] = node.xpath('description').extract() return item Basically what we did up there was to create a spider that downloads a feed from diff --git a/sep/sep-020.rst b/sep/sep-020.rst index e1d563ba7..7b2c043b7 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -91,9 +91,9 @@ This is a working code sample that covers just the basics. """ Pull the text label out of selected markup :param entity: Found markup - :type entity: HtmlXPathSelector + :type entity: Selector """ - label = ' '.join(entity.select('.//text()').extract()) + label = ' '.join(entity.xpath('.//text()').extract()) label = label.encode('ascii', 'xmlcharrefreplace') if label else '' label = label.strip(' ') if ' ' in label else label label = label.strip(':') if ':' in label else label @@ -108,7 +108,7 @@ This is a working code sample that covers just the basics. :return: The list of selectors :rtype: list """ - return self.selector.select(self.base_xpath + xpath) + return self.selector.xpath(self.base_xpath + xpath) def parse_dl(self, xpath=u'//dl'): """ Look for the specified definition list pattern and store all found @@ -120,7 +120,7 @@ This is a working code sample that covers just the basics. for term in self._get_entities(xpath + '/dt'): label = self._get_label(term) if label and label not in self.ignore: - value = term.select('following-sibling::dd[1]//text()') + value = term.xpath('following-sibling::dd[1]//text()') if value: self.add_value(label, value.extract(), MapCompose(lambda v: v.strip()))