From bc17e9d41242784bbf9b145de80a9483e3c7844f Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Sun, 30 Sep 2012 14:55:55 -0300 Subject: [PATCH 01/19] Adds HtmlCSSSelector and XmlCSSSelector classes, cssselect as optional dependency. Ported .get() from _Element and .text_content() from HTMLMixin Add CSS selectors to scrapy shell Documenting CSS Selectors: Constructing selectors Documenting CSS Selectors: Using Selectors Make CSS Selectors a default feature. Adds XPath powers to CSS Selectors and some syntactic sugar. Removes methods copied over from lxml.html.HtmlMixin. Updating docs to use new CSS Selector super powers. Documenting CSS Selectors: Regular Expressions Moving section after Nesting section, since it mentions it. Documenting CSS Selectors: Nesting Selectors Fix XPath specificity in lxml.selector.CSSSelectorMixin.text Cleaning up unused stuff from cssel.py Changing the behavior of lxml.selector.CSSSelectorMixin.text. Concatenating all of the descendant text nodes is more useful than returning it in pieces (there's xpath() if you need that). Documenting CSS Selectors: CSS Selector objects Documenting CSS Selectors: CSSSelectorList objects Documenting CSS Selectors: HtmlCSSSelector objects Documenting CSS Selectors: XmlCSSSelector objects Fixing some documentations typos and errors Enforcing the 80-char width lines Tidying up CSS selectors and CSSSelectorMixin objects Adding some missing references in documentation. Fixing lxml.selector.CSSSelectorList.text --- docs/topics/selectors.rst | 360 ++++++++++++++++++++++++++++-------- scrapy/selector/__init__.py | 2 + scrapy/selector/csssel.py | 32 ++++ scrapy/shell.py | 6 +- setup.py | 2 +- 5 files changed, 325 insertions(+), 77 deletions(-) create mode 100644 scrapy/selector/csssel.py diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index ec704272b..f3a3d5a0d 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -6,22 +6,24 @@ Selectors When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to -achieve this: +achieve this: * `BeautifulSoup`_ is a very popular screen scraping library among Python - programmers which constructs a Python object based on the - structure of the HTML code and also deals with bad markup reasonably well, - but it has one drawback: it's slow. + programmers which constructs a Python object based on the structure of the + HTML code and also deals with bad markup reasonably well, but it has one + drawback: it's slow. * `lxml`_ is a XML parsing library (which also parses HTML) with a pythonic API based on `ElementTree`_ (which is not part of the Python standard library). -Scrapy comes with its own mechanism for extracting data. They're called XPath -selectors (or just "selectors", for short) because they "select" certain parts -of the HTML document specified by `XPath`_ expressions. +Scrapy comes with its own mechanism for extracting data. They're called +selectors because they "select" certain parts of the HTML document specified +either by `XPath`_ or `CSS`_ expressions. -`XPath`_ is a language for selecting nodes in XML documents, which can also be used with HTML. +`XPath`_ is a language for selecting nodes in XML documents, which can also be +used with HTML. `CSS`_ is a language for applying styles to HTML documents. It +defines selectors to associate those styles with specific HTML elements. Both `lxml`_ and Scrapy Selectors are built over the `libxml2`_ library, which means they're very similar in speed and parsing accuracy. @@ -31,14 +33,16 @@ small and simple, unlike the `lxml`_ API which is much bigger because the `lxml`_ library can be used for many other tasks, besides selecting markup documents. -For a complete reference of the selectors API see the :ref:`XPath selector -reference `. +For a complete reference of the selectors API see :ref:`XPath selector +reference ` and :ref:`CSS selector reference +`. .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://codespeak.net/lxml/ .. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html .. _libxml2: http://xmlsoft.org/ .. _XPath: http://www.w3.org/TR/xpath +.. _CSS: http://www.w3.org/TR/selectors Using selectors =============== @@ -46,24 +50,33 @@ Using selectors Constructing selectors ---------------------- -There are two types of selectors bundled with Scrapy. Those are: +There are four types of selectors bundled with Scrapy. Those are: - * :class:`~scrapy.selector.HtmlXPathSelector` - for working with HTML documents + * :class:`~scrapy.selector.HtmlXPathSelector` - for working with HTML + documents using XPath. * :class:`~scrapy.selector.XmlXPathSelector` - for working with XML documents + using XPath. + + * :class:`~scrapy.selector.HtmlCSSSelector` - for working with HTML documents + using CSS selectors. + + * :class:`~scrapy.selector.XmlCSSSelector` - for working with XML documents + using CSS selectors. .. highlight:: python -Both share the same selector API, and are constructed with a Response object as -their first parameter. This is the Response they're going to be "selecting". +All of them share the same selector API, and are constructed with a Response +object as their first parameter. This is the Response they're going to be +"selecting". Example:: - hxs = HtmlXPathSelector(response) # a HTML selector - xxs = XmlXPathSelector(response) # a XML selector + hcs = HtmlCSSSelector(response) # an HTML CSS selector + xxs = XmlXPathSelector(response) # an XML XPath selector -Using selectors with XPaths ---------------------------- +Using selectors +--------------- To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy @@ -84,24 +97,28 @@ First, let's open the shell:: scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html -Then, after the shell loads, you'll have some selectors already instantiated and -ready to use. +Then, after the shell loads, you'll have some selectors already instantiated +and ready to use. -Since we're dealing with HTML, we'll be using the -:class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, in -the ``hxs`` shell variable. +Since we're dealing with HTML, we can use either the +:class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, +in the ``hxs`` shell variable, or the equivalent +:class:`~scrapy.selector.HtmlCSSSelector` found in the ``hcs`` shell variable. +Note that CSS selectors can only select element nodes, while XPath selectors +can select any nodes, including text and comment nodes. There are some methods +to augment CSS selectors with XPath as we'll see below. .. highlight:: python -So, by looking at the :ref:`HTML code ` of that page, -let's construct an XPath (using an HTML selector) for selecting the text inside -the title tag:: +So, by looking at the :ref:`HTML code ` of that +page, let's construct an XPath (using an HTML selector) for selecting the text +inside the title tag:: >>> hxs.select('//title/text()') [] -As you can see, the select() method returns an XPathSelectorList, which is a list of -new selectors. This API can be used quickly for extracting nested data. +As you can see, the select() method returns an XPathSelectorList, which is a +list of new selectors. This API can be used quickly for extracting nested data. To actually extract the textual data, you must call the selector ``extract()`` method, as follows:: @@ -109,11 +126,22 @@ method, as follows:: >>> hxs.select('//title/text()').extract() [u'Example website'] +Now notice that CSS selectors can't select the text nodes. There are some +methods that allow enhancing CSS selectors, such as ``text`` and ``get``:: + + >>> hcs.select('title').text() + [] + >>> hcs.select('title').text().extract() + [u'Example website'] + Now we're going to get the base URL and some image links:: >>> hxs.select('//base/@href').extract() [u'http://example.com/'] + >>> hcs.select('base').get('href') + [u'http://example.com/'] + >>> hxs.select('//a[contains(@href, "image")]/@href').extract() [u'image1.html', u'image2.html', @@ -121,6 +149,13 @@ Now we're going to get the base URL and some image links:: u'image4.html', u'image5.html'] + >>> hcs.select('a[href*=image]').get('href').extract() + [u'image1.html', + u'image2.html', + u'image3.html', + u'image4.html', + u'image5.html'] + >>> hxs.select('//a[contains(@href, "image")]/img/@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', @@ -128,32 +163,21 @@ Now we're going to get the base URL and some image links:: u'image4_thumb.jpg', u'image5_thumb.jpg'] - -Using selectors with regular expressions ----------------------------------------- - -Selectors also have a ``re()`` method for extracting data using regular -expressions. However, unlike using the ``select()`` method, the ``re()`` method -does not return a list of :class:`~scrapy.selector.XPathSelector` objects, so you -can't construct nested ``.re()`` calls. - -Here's an example used to extract images names from the :ref:`HTML code -` above:: - - >>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') - [u'My image 1', - u'My image 2', - u'My image 3', - u'My image 4', - u'My image 5'] + >>> hcs.select('a[href*=image] img').get('src').extract() + [u'image1_thumb.jpg', + u'image2_thumb.jpg', + u'image3_thumb.jpg', + u'image4_thumb.jpg', + u'image5_thumb.jpg'] .. _topics-selectors-nesting-selectors: Nesting selectors ----------------- -The ``select()`` selector method returns a list of selectors, so you can call the -``select()`` for those selectors too. Here's an example:: +The ``select()`` selector method returns a list of selectors of the same type +(XPath or CSS), so you can call the ``select()`` for those selectors too. +Here's an example:: >>> links = hxs.select('//a[contains(@href, "image")]') >>> links.extract() @@ -173,6 +197,40 @@ The ``select()`` selector method returns a list of selectors, so you can call th Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] +The CSSSelectorList ``select`` method will accept CSS selectors, as expected, +but it also provides an ``xpath`` method that accepts XPath selectors to +augment the CSS selectors. Here's an example:: + + >>> links = hcs.select('a[href*=image]') + >>> for index, link in enumerate(links): + args = (index, link.get('href').extract(), link.xpath('img/@src').extract()) + print 'Link number %d points to url %s and image %s' % args + + Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] + Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg'] + Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg'] + Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] + Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] + +Using selectors with regular expressions +---------------------------------------- + +Selectors (both CSS and XPath) also have a ``re()`` method for extracting data +using regular expressions. However, unlike using the ``select()`` method, the +``re()`` method does not return a list of +:class:`~scrapy.selector.XPathSelector` objects, so you can't construct nested +``.re()`` calls. + +Here's an example used to extract images names from the :ref:`HTML code +` above:: + + >>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + [u'My image 1', + u'My image 2', + u'My image 3', + u'My image 4', + u'My image 5'] + .. _topics-selectors-relative-xpaths: Working with relative XPaths @@ -212,16 +270,19 @@ XPath specification. .. _topics-selectors-ref: -Built-in XPath Selectors reference +Built-in Selectors reference ================================== .. module:: scrapy.selector - :synopsis: XPath selectors classes + :synopsis: Selectors classes -There are two types of selectors bundled with Scrapy: -:class:`HtmlXPathSelector` and :class:`XmlXPathSelector`. Both of them -implement the same :class:`XPathSelector` interface. The only different is that -one is used to process HTML data and the other XML data. +There are four types of selectors bundled with Scrapy: +:class:`HtmlXPathSelector` and :class:`XmlXPathSelector`, +:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. All of them implement the +same :class:`XPathSelector` interface. The only differences are the selector +syntax and whether it is used to process HTML data or XML data. + +.. _topics-xpath-selectors-ref: XPathSelector objects --------------------- @@ -232,13 +293,13 @@ XPathSelector objects certain parts of its content. ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data + for selecting and extracting data. .. method:: select(xpath) Apply the given XPath relative to this XPathSelector and return a list - of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) with - the result. + of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) + with the result. ``xpath`` is a string containing the XPath to apply @@ -269,8 +330,8 @@ XPathSelector objects .. method:: __nonzero__() Returns ``True`` if there is any real content selected by this - :class:`XPathSelector` or ``False`` otherwise. In other words, the boolean - value of an XPathSelector is given by the contents it selects. + :class:`XPathSelector` or ``False`` otherwise. In other words, the + boolean value of an XPathSelector is given by the contents it selects. XPathSelectorList objects ------------------------- @@ -282,11 +343,12 @@ XPathSelectorList objects .. method:: select(xpath) - Call the :meth:`XPathSelector.select` method for all :class:`XPathSelector` - objects in this list and return their results flattened, as a new - :class:`XPathSelectorList`. + Call the :meth:`XPathSelector.select` method for all + :class:`XPathSelector` objects in this list and return their results + flattened, as a new :class:`XPathSelectorList`. - ``xpath`` is the same argument as the one in :meth:`XPathSelector.select` + ``xpath`` is the same argument as the one in + :meth:`XPathSelector.select` .. method:: re(regex) @@ -298,16 +360,16 @@ XPathSelectorList objects .. method:: extract() - Call the :meth:`XPathSelector.extract` method for all :class:`XPathSelector` - objects in this list and return their results flattened, as a list of - unicode strings. + Call the :meth:`XPathSelector.extract` method for all + :class:`XPathSelector` objects in this list and return their results + flattened, as a list of unicode strings. .. method:: extract_unquoted() Call the :meth:`XPathSelector.extract_unoquoted` method for all :class:`XPathSelector` objects in this list and return their results - flattened, as a list of unicode strings. This method should not be applied - to all kinds of XPathSelectors. For more info see + flattened, as a list of unicode strings. This method should not be + applied to all kinds of XPathSelectors. For more info see :meth:`XPathSelector.extract_unoquoted`. HtmlXPathSelector objects @@ -316,7 +378,8 @@ HtmlXPathSelector objects .. class:: HtmlXPathSelector(response) A subclass of :class:`XPathSelector` for working with HTML content. It uses - the `libxml2`_ HTML parser. See the :class:`XPathSelector` API for more info. + the `libxml2`_ HTML parser. See the :class:`XPathSelector` API for more + info. .. _libxml2: http://xmlsoft.org/ @@ -324,8 +387,9 @@ HtmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`HtmlXPathSelector` examples to illustrate several -concepts. In all cases, we assume there is already an :class:`HtmlPathSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: +concepts. In all cases, we assume there is already an +:class:`HtmlXPathSelector` instantiated with a :class:`~scrapy.http.Response` +object like this:: x = HtmlXPathSelector(html_response) @@ -343,7 +407,7 @@ instantiated with a :class:`~scrapy.http.Response` object like this:: 3. Iterate over all ``

`` tags and print their class attribute:: for node in x.select("//p"): - ... print node.select("@href") + ... print node.select("@class").extract() 4. Extract textual data from all ``

`` tags without entities, as a list of unicode strings:: @@ -366,13 +430,14 @@ XmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`XmlXPathSelector` examples to illustrate several -concepts. In both cases we assume there is already an :class:`XmlXPathSelector` +concepts. In both cases we assume there is already an :class:`XmlXPathSelector` instantiated with a :class:`~scrapy.http.Response` object like this:: x = XmlXPathSelector(xml_response) -1. Select all ```` elements from a XML response body, returning a list of - :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object):: +1. Select all ```` elements from a XML response body, returning a list + of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` + object):: x.select("//product") @@ -426,3 +491,148 @@ of relevance, are: though. .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 + +.. _topics-css-selectors-ref: + +CSSSelector objects +-------------------- + +.. class:: CSSSelectorMixin(object) + + A :class:`CSSSelectorMixin` object is a mixin for either + :class:`XmlXPathSelector` or :class:`HtmlXPathSelector` to select element + nodes using CSS selectors syntax. As a mixin, it is not meant to be used on + its own, but as a secondary parent class. See :class:`XmlCSSSelector` and + :class:`HtmlCSSSelector` for implementations. + + .. method:: select(css) + + Apply the given CSS selector relative to this CSSSelectorMixin and + return a list of :class:`CSSSelectorMixin` objects (ie. a + :class:`CSSSelectorList`) with the result. + + ``css`` is a string containing the CSS selector to apply. + + .. method:: xpath(xpath) + + Apply the given XPath relative to this CSSSelectorMixin and return a list + of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) + with the result. + + ``xpath`` is a string containing the XPath to apply. + + .. method:: get(attr) + + Get the attribute relative to this CSSSelectorMixin and return a list + of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) + with the result (usually with one element only). + + ``attr`` is a string containing the attribute name to get. + + .. method:: text(all=False) + + Get the children text nodes relative to this CSSSelectorMixin or, if + ``all`` is True, a string node concatenating all of the descendant text + nodes relative to this CSSSelectorMixin, and return a list of + :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) with + the result. + + ``all`` is a boolean to either select children text nodes (False) or + select a string node concatenating all of the descendant text nodes. + +CSSSelectorList objects +----------------------- + +.. class:: CSSSelectorList + + The :class:`CSSSelectorList` class is subclass of :class:`XPathSelectorList` + which overrides and adds methods to match those of + :class:`CSSSelectorMixin`. + + .. method:: xpath(xpath) + + Call the :meth:`CSSSelectorMixin.xpath` method for all + :class:`CSSSelectorMixin` objects in this list and return their results + flattened, as a new :class:`CSSSelectorList`. + + ``xpath`` is the same argument as the one in + :meth:`CSSSelectorMixin.xpath` + + .. method:: get(attr) + + Call the :meth:`CSSSelectorMixin.get` method for all + :class:`CSSSelectorMixin` objects in this list and return their results + flattened, as a new :class:`CSSSelectorList`. + + ``attr`` is the same argument as the one in :meth:`CSSSelectorMixin.get` + + .. method:: text(all=False) + + Call the :meth:`CSSSelectorMixin.text` method for all + :class:`CSSSelectorMixin` objects in this list and return their results + flattened, as a new :class:`CSSSelectorList`. + + ``all`` is the same argument as the one in :meth:`CSSSelectorMixin.text` + +HtmlCSSSelector objects +----------------------- + +.. class:: HtmlCSSSelector(response) + + A subclass of :class:`CSSSelectorMixin` and :class:`HtmlXPathSelector` for + working with HTML content using CSS selectors. + +HtmlCSSSelector examples +~~~~~~~~~~~~~~~~~~~~~~~~ + +Here's a couple of :class:`HtmlCSSSelector` examples to illustrate several +concepts. In all cases, we assume there is already an :class:`HtmlCSSSelector` +instantiated with a :class:`~scrapy.http.Response` object like this:: + + x = HtmlCSSSelector(html_response) + +1. Select all ``

`` elements from a HTML response body, returning a list of + :class:`HtmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + + x.select("h1") + +2. Extract the text of all ``

`` elements from a HTML response body, + returning a list of unicode strings:: + + x.select("h1").extract() # this includes the h1 tag + x.select("h1").text().extract() # this excludes the h1 tag + +3. Iterate over all ``

`` tags and print their class attribute:: + + for node in x.select("p"): + ... print node.get("class").extract() + +XmlCSSSelector objects +---------------------- + +.. class:: XmlCSSSelector(response) + + A subclass of :class:`CSSSelectorMixin` and :class:`XmlXPathSelector` for + working with XML content using CSS selectors. + +XmlCSSSelector examples +~~~~~~~~~~~~~~~~~~~~~~~ + +Here's a couple of :class:`XmlCSSSelector` examples to illustrate several +concepts. In both cases we assume there is already an :class:`XmlCSSSelector` +instantiated with a :class:`~scrapy.http.Response` object like this:: + + x = XmlCSSSelector(xml_response) + +1. Select all ```` elements from a XML response body, returning a list + of :class:`XmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + + x.select("product") + +2. Extract all prices from a `Google Base XML feed`_ which requires registering + a namespace:: + + x.register_namespace("g", "http://base.google.com/ns/1.0") + x.xpath("//g:price").extract() + +.. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 239a3091e..86ece8f0e 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -24,3 +24,5 @@ else: from scrapy.selector.libxml2sel import * else: from scrapy.selector.lxmlsel import * + +from scrapy.selector.csssel import * diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py new file mode 100644 index 000000000..9fce75877 --- /dev/null +++ b/scrapy/selector/csssel.py @@ -0,0 +1,32 @@ +from cssselect import GenericTranslator, HTMLTranslator +from scrapy.utils.python import flatten +from scrapy.selector import HtmlXPathSelector, XmlXPathSelector, XPathSelectorList + +class CSSSelectorList(XPathSelectorList): + def xpath(self, xpath): + return self.__class__(flatten([x.xpath(xpath) for x in self])) + + def get(self, attr): + return self.__class__(flatten([x.get(attr) for x in self])) + + def text(self, all=False): + return self.__class__(flatten([x.text(all) for x in self])) + +class CSSSelectorMixin(object): + def select(self, css): + return CSSSelectorList(super(CSSSelectorMixin, self).select(self.translator.css_to_xpath(css))) + + def xpath(self, xpath): + return CSSSelectorList(super(CSSSelectorMixin, self).select(xpath)) + + def text(self, all=False): + return self.xpath('string()') if all else self.xpath('text()') + + def get(self, attr): + return self.xpath('@' + attr) + +class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): + translator = GenericTranslator() + +class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector): + translator = HTMLTranslator() diff --git a/scrapy/shell.py b/scrapy/shell.py index ac6675f8e..642b3d6e6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -11,7 +11,7 @@ from w3lib.url import any_to_uri from scrapy.item import BaseItem from scrapy.spider import BaseSpider -from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector +from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector, XmlCSSSelector, HtmlCSSSelector from scrapy.utils.spider import create_spider_for_request from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser @@ -97,8 +97,12 @@ class Shell(object): self.vars['response'] = response self.vars['xxs'] = XmlXPathSelector(response) \ if isinstance(response, XmlResponse) else None + self.vars['xcs'] = XmlCSSSelector(response) \ + if isinstance(response, XmlResponse) else None self.vars['hxs'] = HtmlXPathSelector(response) \ if isinstance(response, HtmlResponse) else None + self.vars['hcs'] = HtmlCSSSelector(response) \ + if isinstance(response, HtmlResponse) else None if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser diff --git a/setup.py b/setup.py index 1b67133c4..bc2edea0c 100644 --- a/setup.py +++ b/setup.py @@ -122,6 +122,6 @@ try: except ImportError: from distutils.core import setup else: - setup_args['install_requires'] = ['Twisted>=10.0.0', 'w3lib>=1.2', 'queuelib', 'lxml', 'pyOpenSSL'] + setup_args['install_requires'] = ['Twisted>=10.0.0', 'w3lib>=1.2', 'queuelib', 'lxml', 'pyOpenSSL', 'cssselect>0.8'] setup(**setup_args) From b38ac27eee70b41a4d6d2545b5bf78139ff23f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 9 Jan 2013 11:09:02 -0200 Subject: [PATCH 02/19] rename XPathSelectorList as SelectorList #176 --- scrapy/selector/__init__.py | 15 +++++++++++++-- scrapy/selector/csssel.py | 9 +++++++-- scrapy/selector/libxml2sel.py | 15 ++++++++------- scrapy/selector/list.py | 5 +++-- scrapy/selector/lxmlsel.py | 15 ++++++--------- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 86ece8f0e..2b855d147 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -7,11 +7,10 @@ variable. Two backends are currently available: lxml (default) and libxml2. """ - import os -backend = os.environ.get('SCRAPY_SELECTORS_BACKEND') +backend = os.environ.get('SCRAPY_SELECTORS_BACKEND') if backend == 'libxml2': from scrapy.selector.libxml2sel import * elif backend == 'lxml': @@ -26,3 +25,15 @@ else: from scrapy.selector.lxmlsel import * from scrapy.selector.csssel import * +from scrapy.selector.list import SelectorList + + +class XPathSelectorList(SelectorList): + + def __init__(self, *a, **kw): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('XPathSelectorList is deprecated, use ' + 'scrapy.selector.SelectorList instead', + category=ScrapyDeprecationWarning, stacklevel=1) + super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index 9fce75877..13da8df95 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,8 +1,10 @@ from cssselect import GenericTranslator, HTMLTranslator from scrapy.utils.python import flatten -from scrapy.selector import HtmlXPathSelector, XmlXPathSelector, XPathSelectorList +from scrapy.selector import HtmlXPathSelector, XmlXPathSelector +from .list import SelectorList -class CSSSelectorList(XPathSelectorList): + +class CSSSelectorList(SelectorList): def xpath(self, xpath): return self.__class__(flatten([x.xpath(xpath) for x in self])) @@ -12,6 +14,7 @@ class CSSSelectorList(XPathSelectorList): def text(self, all=False): return self.__class__(flatten([x.text(all) for x in self])) + class CSSSelectorMixin(object): def select(self, css): return CSSSelectorList(super(CSSSelectorMixin, self).select(self.translator.css_to_xpath(css))) @@ -25,8 +28,10 @@ class CSSSelectorMixin(object): def get(self, attr): return self.xpath('@' + attr) + class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): translator = GenericTranslator() + class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector): translator = HTMLTranslator() diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py index da546ade2..2f5d72ea1 100644 --- a/scrapy/selector/libxml2sel.py +++ b/scrapy/selector/libxml2sel.py @@ -12,10 +12,11 @@ from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml -from .list import XPathSelectorList +from .list import SelectorList + + +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] class XPathSelector(object_ref): @@ -44,13 +45,13 @@ class XPathSelector(object_ref): except libxml2.xpathError: raise ValueError("Invalid XPath: %s" % xpath) if hasattr(xpath_result, '__iter__'): - return XPathSelectorList([self.__class__(node=node, parent=self, \ + return SelectorList([self.__class__(node=node, parent=self, \ expr=xpath) for node in xpath_result]) else: - return XPathSelectorList([self.__class__(node=xpath_result, \ + return SelectorList([self.__class__(node=xpath_result, \ parent=self, expr=xpath)]) else: - return XPathSelectorList([]) + return SelectorList([]) def re(self, regex): return extract_regex(regex, self.extract()) @@ -62,7 +63,7 @@ class XPathSelector(object_ref): if isinstance(self.xmlNode, libxml2.xmlDoc): data = self.xmlNode.getRootElement().serialize('utf-8') text = unicode(data, 'utf-8', errors='ignore') if data else u'' - elif isinstance(self.xmlNode, libxml2.xmlAttr): + elif isinstance(self.xmlNode, libxml2.xmlAttr): # serialization doesn't work sometimes for xmlAttr types text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') else: diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py index a6977fd99..ea634db1f 100644 --- a/scrapy/selector/list.py +++ b/scrapy/selector/list.py @@ -1,7 +1,8 @@ from scrapy.utils.python import flatten from scrapy.utils.decorator import deprecated -class XPathSelectorList(list): + +class SelectorList(list): def __getslice__(self, i, j): return self.__class__(list.__getslice__(self, i, j)) @@ -18,6 +19,6 @@ class XPathSelectorList(list): def extract_unquoted(self): return [x.extract_unquoted() for x in self] - @deprecated(use_instead='XPathSelectorList.select') + @deprecated(use_instead='SelectorList.select') def x(self, xpath): return self.select(xpath) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 5bf9fd1dc..18b3e5044 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -10,11 +10,10 @@ from scrapy.utils.python import unicode_to_str from scrapy.utils.decorator import deprecated from scrapy.http import TextResponse from .lxmldocument import LxmlDocument -from .list import XPathSelectorList +from .list import SelectorList -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] class XPathSelector(object_ref): @@ -25,8 +24,8 @@ class XPathSelector(object_ref): def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None): if text is not None: - response = TextResponse(url='about:blank', \ - body=unicode_to_str(text, 'utf-8'), encoding='utf-8') + response = TextResponse(url='about:blank', encoding='utf-8', + body=unicode_to_str(text, 'utf-8')) if response is not None: _root = LxmlDocument(response, self._parser) @@ -39,7 +38,7 @@ class XPathSelector(object_ref): try: xpathev = self._root.xpath except AttributeError: - return XPathSelectorList([]) + return SelectorList([]) try: result = xpathev(xpath, namespaces=self.namespaces) @@ -51,7 +50,7 @@ class XPathSelector(object_ref): result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces) for x in result] - return XPathSelectorList(result) + return SelectorList(result) def re(self, regex): return extract_regex(regex, self.extract()) @@ -88,10 +87,8 @@ class XPathSelector(object_ref): def __str__(self): data = repr(self.extract()[:40]) return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) - __repr__ = __str__ - @deprecated(use_instead='XPathSelector.extract') def extract_unquoted(self): return self.extract() From 4e6967b854644d527c0eb5ace61e80a590fbd520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 9 Jan 2013 17:48:20 -0200 Subject: [PATCH 03/19] extend css selectors with ":text" and :attribute() #176 --- scrapy/selector/csssel.py | 95 ++++++++++++++----- scrapy/tests/test_selector_cssselect.py | 117 ++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 scrapy/tests/test_selector_cssselect.py diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index 13da8df95..d5b571bae 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,37 +1,88 @@ from cssselect import GenericTranslator, HTMLTranslator -from scrapy.utils.python import flatten -from scrapy.selector import HtmlXPathSelector, XmlXPathSelector -from .list import SelectorList +from cssselect.xpath import XPathExpr, ExpressionError +from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector -class CSSSelectorList(SelectorList): - def xpath(self, xpath): - return self.__class__(flatten([x.xpath(xpath) for x in self])) +class ScrapyXPathExpr(XPathExpr): - def get(self, attr): - return self.__class__(flatten([x.get(attr) for x in self])) + textnode = False + attribute = None - def text(self, all=False): - return self.__class__(flatten([x.text(all) for x in self])) + @classmethod + def from_xpath(cls, xpath, textnode=False, attribute=None): + x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) + x.textnode = textnode + x.attribute = attribute + return x + + def __str__(self): + path = super(ScrapyXPathExpr, self).__str__() + if self.textnode: + if path == '*': + path = 'text()' + elif path.endswith('::*/*'): + path = path[:-3] + 'text()' + else: + path += '/text()' + + if self.attribute is not None: + if path.endswith('::*/*'): + path = path[:-2] + path += '/@%s' % self.attribute + + return path + + def join(self, combiner, other): + super(ScrapyXPathExpr, self).join(combiner, other) + self.textnode = other.textnode + self.attribute = other.attribute + return self + + +class TranslatorMixin(object): + + def xpath_element(self, selector): + xpath = super(TranslatorMixin, self).xpath_element(selector) + return ScrapyXPathExpr.from_xpath(xpath) + + def xpath_text_pseudo(self, xpath): + """Support selecting text nodes using :text pseudo-element""" + return ScrapyXPathExpr.from_xpath(xpath, textnode=True) + + def xpath_attribute_function(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :contains(), got %r" + % function.arguments) + value = function.arguments[0].value + return ScrapyXPathExpr.from_xpath(xpath, attribute=value) + + +class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): + pass + + +class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator): + pass class CSSSelectorMixin(object): + def select(self, css): - return CSSSelectorList(super(CSSSelectorMixin, self).select(self.translator.css_to_xpath(css))) + xpath = self._css2xpath(css) + return super(CSSSelectorMixin, self).select(xpath) - def xpath(self, xpath): - return CSSSelectorList(super(CSSSelectorMixin, self).select(xpath)) - - def text(self, all=False): - return self.xpath('string()') if all else self.xpath('text()') - - def get(self, attr): - return self.xpath('@' + attr) + def _css2xpath(self, css): + return self.translator.css_to_xpath(css) -class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): - translator = GenericTranslator() +class CSSSelector(CSSSelectorMixin, XPathSelector): + translator = ScrapyHTMLTranslator() class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector): - translator = HTMLTranslator() + translator = ScrapyHTMLTranslator() + + +class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): + translator = ScrapyGenericTranslator() diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py new file mode 100644 index 000000000..f406a80ee --- /dev/null +++ b/scrapy/tests/test_selector_cssselect.py @@ -0,0 +1,117 @@ +""" +Selector tests for cssselect backend +""" +from twisted.trial import unittest +from scrapy.http import TextResponse, HtmlResponse, XmlResponse +from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector +from scrapy.selector.csssel import ScrapyHTMLTranslator + +HTMLBODY = ''' + + +

+ + + link +

+ lorem ipsum text + hi there + guy + + + + + + + +

+ + +
+

+ + + + +
+ + +''' + + +class TranslatorMixinTest(unittest.TestCase): + + tr_cls = ScrapyHTMLTranslator + + def setUp(self): + self.tr = self.tr_cls() + self.c2x = self.tr.css_to_xpath + + def test_attribute_function(self): + cases = [ + (':attribute(name)', u'descendant-or-self::*/@name'), + ('a:attribute(name)', u'descendant-or-self::a/@name'), + ('a :attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), + ('a > :attribute(name)', u'descendant-or-self::a/*/@name'), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) + + def test_text_pseudo_element(self): + cases = [ + (':text', u'descendant-or-self::text()'), + ('p:text', u'descendant-or-self::p/text()'), + ('p :text', u'descendant-or-self::p/descendant-or-self::text()'), + ('#id:text', u"descendant-or-self::*[@id = 'id']/text()"), + ('p#id:text', u"descendant-or-self::p[@id = 'id']/text()"), + ('p#id :text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"), + ('p#id > :text', u"descendant-or-self::p[@id = 'id']/*/text()"), + ('p#id ~ :text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), + ('a[href]:text', u'descendant-or-self::a[@href]/text()'), + ('a[href] :text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), + ('p:text, a:text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) + + +class HTMLCSSSelectorTest(unittest.TestCase): + + hcs_cls = HtmlCSSSelector + + def setUp(self): + self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY) + self.hcs = self.hcs_cls(self.htmlresponse) + + def x(self, *a, **kw): + return [v.strip() for v in self.hcs.select(*a, **kw).extract() if v.strip()] + + def test_selector_simple(self): + for x in self.hcs.select('input'): + self.assertTrue(isinstance(x, self.hcs.__class__), x) + self.assertEqual(self.hcs.select('input').extract(), + [x.extract() for x in self.hcs.select('input')]) + + def test_text_pseudo_element(self): + self.assertEqual(self.x('#p-b2'), [u'guy']) + self.assertEqual(self.x('#p-b2:text'), [u'guy']) + self.assertEqual(self.x('#p-b2 :text'), [u'guy']) + self.assertEqual(self.x('#paragraph:text'), [u'lorem ipsum text']) + self.assertEqual(self.x('#paragraph :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + self.assertEqual(self.x('p:text'), [u'lorem ipsum text']) + self.assertEqual(self.x('p :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + + def test_attribute_function(self): + self.assertEqual(self.x('#p-b2:attribute(id)'), [u'p-b2']) + self.assertEqual(self.x('.cool-footer:attribute(class)'), [u'cool-footer']) + self.assertEqual(self.x('.cool-footer :attribute(id)'), [u'foobar-div', u'foobar-span']) + self.assertEqual(self.x('map[name="dummymap"] :attribute(shape)'), [u'circle', u'default']) + + def test_nested_selector(self): + self.assertEqual(self.hcs.select('p').select('b:text').extract(), + [u'hi', u'guy']) + self.assertEqual(self.hcs.select('div').select('area:last-child').extract(), + [u'']) From 52e2eb6b703fbef709bdb5250758466e17862eee Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 16 Sep 2013 13:26:34 +0200 Subject: [PATCH 04/19] Adapt to latest cssselect API supporting pseudo-elements --- scrapy/selector/csssel.py | 30 ++++++++++++++++++++++++- scrapy/tests/test_selector_cssselect.py | 26 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index d5b571bae..90a3261df 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,5 +1,6 @@ from cssselect import GenericTranslator, HTMLTranslator -from cssselect.xpath import XPathExpr, ExpressionError +from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError +from cssselect.parser import FunctionalPseudoElement from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector @@ -57,6 +58,33 @@ class TranslatorMixin(object): value = function.arguments[0].value return ScrapyXPathExpr.from_xpath(xpath, attribute=value) + def xpath_pseudo_element(self, xpath, pseudo_element): + if isinstance(pseudo_element, FunctionalPseudoElement): + method = 'xpath_%s_functional_pseudo_element' % ( + pseudo_element.name.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The functional pseudo-element ::%s() is unknown" + % pseudo_element.name) + xpath = method(xpath, pseudo_element) + else: + method = 'xpath_%s_simple_pseudo_element' % ( + pseudo_element.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The pseudo-element ::%s is unknown" + % pseudo_element) + xpath = method(xpath) + return xpath + + def xpath_attribute_functional_pseudo_element(self, xpath, arguments): + return self.xpath_attribute_function(xpath, arguments) + + def xpath_text_simple_pseudo_element(self, xpath): + return self.xpath_text_pseudo(xpath) + class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): pass diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index f406a80ee..058ebc783 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -60,6 +60,16 @@ class TranslatorMixinTest(unittest.TestCase): for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_attribute_function2(self): + cases = [ + ('::attribute(name)', u'descendant-or-self::*/@name'), + ('a::attribute(name)', u'descendant-or-self::a/@name'), + ('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), + ('a > ::attribute(name)', u'descendant-or-self::a/*/@name'), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) + def test_text_pseudo_element(self): cases = [ (':text', u'descendant-or-self::text()'), @@ -77,6 +87,22 @@ class TranslatorMixinTest(unittest.TestCase): for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_text_pseudo_element2(self): + cases = [ + ('::text', u'descendant-or-self::text()'), + ('p::text', u'descendant-or-self::p/text()'), + ('p ::text', u'descendant-or-self::p/descendant-or-self::text()'), + ('#id::text', u"descendant-or-self::*[@id = 'id']/text()"), + ('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"), + ('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"), + ('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"), + ('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), + ('a[href]::text', u'descendant-or-self::a[@href]/text()'), + ('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), + ('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) class HTMLCSSSelectorTest(unittest.TestCase): From ea579bca0456f4e0da656af7c39d09b793f417ef Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 16 Sep 2013 15:36:35 +0200 Subject: [PATCH 05/19] Support only ::text and ::attr(name) + add more CSS selector tests --- scrapy/selector/csssel.py | 24 +++---- scrapy/tests/test_selector_cssselect.py | 92 ++++++++++++++----------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index 90a3261df..e8c5538aa 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -46,18 +46,6 @@ class TranslatorMixin(object): xpath = super(TranslatorMixin, self).xpath_element(selector) return ScrapyXPathExpr.from_xpath(xpath) - def xpath_text_pseudo(self, xpath): - """Support selecting text nodes using :text pseudo-element""" - return ScrapyXPathExpr.from_xpath(xpath, textnode=True) - - def xpath_attribute_function(self, xpath, function): - if function.argument_types() not in (['STRING'], ['IDENT']): - raise ExpressionError( - "Expected a single string or ident for :contains(), got %r" - % function.arguments) - value = function.arguments[0].value - return ScrapyXPathExpr.from_xpath(xpath, attribute=value) - def xpath_pseudo_element(self, xpath, pseudo_element): if isinstance(pseudo_element, FunctionalPseudoElement): method = 'xpath_%s_functional_pseudo_element' % ( @@ -79,11 +67,17 @@ class TranslatorMixin(object): xpath = method(xpath) return xpath - def xpath_attribute_functional_pseudo_element(self, xpath, arguments): - return self.xpath_attribute_function(xpath, arguments) + def xpath_attr_functional_pseudo_element(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for ::attr(), got %r" + % function.arguments) + return ScrapyXPathExpr.from_xpath(xpath, + attribute=function.arguments[0].value) def xpath_text_simple_pseudo_element(self, xpath): - return self.xpath_text_pseudo(xpath) + """Support selecting text nodes using ::text pseudo-element""" + return ScrapyXPathExpr.from_xpath(xpath, textnode=True) class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index 058ebc783..4b69dff15 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -5,6 +5,9 @@ from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector from scrapy.selector.csssel import ScrapyHTMLTranslator +from cssselect.parser import SelectorSyntaxError +from cssselect.xpath import ExpressionError + HTMLBODY = ''' @@ -50,44 +53,26 @@ class TranslatorMixinTest(unittest.TestCase): self.tr = self.tr_cls() self.c2x = self.tr.css_to_xpath - def test_attribute_function(self): + def test_attr_function(self): cases = [ - (':attribute(name)', u'descendant-or-self::*/@name'), - ('a:attribute(name)', u'descendant-or-self::a/@name'), - ('a :attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), - ('a > :attribute(name)', u'descendant-or-self::a/*/@name'), + ('::attr(name)', u'descendant-or-self::*/@name'), + ('a::attr(href)', u'descendant-or-self::a/@href'), + ('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'), + ('a > ::attr(class)', u'descendant-or-self::a/*/@class'), ] for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) - def test_attribute_function2(self): + def test_attr_function_exception(self): cases = [ - ('::attribute(name)', u'descendant-or-self::*/@name'), - ('a::attribute(name)', u'descendant-or-self::a/@name'), - ('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), - ('a > ::attribute(name)', u'descendant-or-self::a/*/@name'), + ('::attr(12)', ExpressionError), + ('::attr(34test)', ExpressionError), + ('::attr(@href)', SelectorSyntaxError), ] - for css, xpath in cases: - self.assertEqual(self.c2x(css), xpath, css) + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) def test_text_pseudo_element(self): - cases = [ - (':text', u'descendant-or-self::text()'), - ('p:text', u'descendant-or-self::p/text()'), - ('p :text', u'descendant-or-self::p/descendant-or-self::text()'), - ('#id:text', u"descendant-or-self::*[@id = 'id']/text()"), - ('p#id:text', u"descendant-or-self::p[@id = 'id']/text()"), - ('p#id :text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"), - ('p#id > :text', u"descendant-or-self::p[@id = 'id']/*/text()"), - ('p#id ~ :text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), - ('a[href]:text', u'descendant-or-self::a[@href]/text()'), - ('a[href] :text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), - ('p:text, a:text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), - ] - for css, xpath in cases: - self.assertEqual(self.c2x(css), xpath, css) - - def test_text_pseudo_element2(self): cases = [ ('::text', u'descendant-or-self::text()'), ('p::text', u'descendant-or-self::p/text()'), @@ -99,11 +84,36 @@ class TranslatorMixinTest(unittest.TestCase): ('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), ('a[href]::text', u'descendant-or-self::a[@href]/text()'), ('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), - ('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), + ('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), ] for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_pseudo_function_exception(self): + cases = [ + ('::attribute(12)', ExpressionError), + ('::text()', ExpressionError), + ('::attr(@href)', SelectorSyntaxError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + def test_unknown_pseudo_element(self): + cases = [ + ('::text-node', ExpressionError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + def test_unknown_pseudo_class(self): + cases = [ + (':text', ExpressionError), + (':attribute(name)', ExpressionError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + class HTMLCSSSelectorTest(unittest.TestCase): hcs_cls = HtmlCSSSelector @@ -123,21 +133,21 @@ class HTMLCSSSelectorTest(unittest.TestCase): def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) - self.assertEqual(self.x('#p-b2:text'), [u'guy']) - self.assertEqual(self.x('#p-b2 :text'), [u'guy']) - self.assertEqual(self.x('#paragraph:text'), [u'lorem ipsum text']) - self.assertEqual(self.x('#paragraph :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) - self.assertEqual(self.x('p:text'), [u'lorem ipsum text']) - self.assertEqual(self.x('p :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + self.assertEqual(self.x('#p-b2::text'), [u'guy']) + self.assertEqual(self.x('#p-b2 ::text'), [u'guy']) + self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text']) + self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + self.assertEqual(self.x('p::text'), [u'lorem ipsum text']) + self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) def test_attribute_function(self): - self.assertEqual(self.x('#p-b2:attribute(id)'), [u'p-b2']) - self.assertEqual(self.x('.cool-footer:attribute(class)'), [u'cool-footer']) - self.assertEqual(self.x('.cool-footer :attribute(id)'), [u'foobar-div', u'foobar-span']) - self.assertEqual(self.x('map[name="dummymap"] :attribute(shape)'), [u'circle', u'default']) + self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2']) + self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer']) + self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span']) + self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.hcs.select('p').select('b:text').extract(), + self.assertEqual(self.hcs.select('p').select('b::text').extract(), [u'hi', u'guy']) self.assertEqual(self.hcs.select('div').select('area:last-child').extract(), [u'']) From 6d598f0d94a7b8e8f121e1149c91151ef7c47060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 24 Sep 2013 12:34:46 -0300 Subject: [PATCH 06/19] Update selectors docs --- docs/topics/selectors.rst | 283 ++++++++++++++------------------------ scrapy/selector/list.py | 1 + 2 files changed, 108 insertions(+), 176 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index f3a3d5a0d..cd4d825d9 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -25,8 +25,8 @@ either by `XPath`_ or `CSS`_ expressions. used with HTML. `CSS`_ is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements. -Both `lxml`_ and Scrapy Selectors are built over the `libxml2`_ library, which -means they're very similar in speed and parsing accuracy. +Scrapy selectors are built over the `lxml`_ library, which means they're very +similar in speed and parsing accuracy. This page explains how selectors work and describes their API which is very small and simple, unlike the `lxml`_ API which is much bigger because the @@ -41,6 +41,7 @@ reference ` and :ref:`CSS selector reference .. _lxml: http://codespeak.net/lxml/ .. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html .. _libxml2: http://xmlsoft.org/ +.. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors @@ -72,8 +73,8 @@ object as their first parameter. This is the Response they're going to be Example:: - hcs = HtmlCSSSelector(response) # an HTML CSS selector - xxs = XmlXPathSelector(response) # an XML XPath selector + hcs = HtmlCSSSelector(response) # an HTML CSS selector + xxs = XmlXPathSelector(response) # an XML XPath selector Using selectors --------------- @@ -104,9 +105,6 @@ Since we're dealing with HTML, we can use either the :class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, in the ``hxs`` shell variable, or the equivalent :class:`~scrapy.selector.HtmlCSSSelector` found in the ``hcs`` shell variable. -Note that CSS selectors can only select element nodes, while XPath selectors -can select any nodes, including text and comment nodes. There are some methods -to augment CSS selectors with XPath as we'll see below. .. highlight:: python @@ -117,8 +115,9 @@ inside the title tag:: >>> hxs.select('//title/text()') [] -As you can see, the select() method returns an XPathSelectorList, which is a -list of new selectors. This API can be used quickly for extracting nested data. +As you can see, the ``select()`` method returns an +:class:`~scrapy.selector.SelectorList`, which is a list of new selectors. This +API can be used quickly for extracting nested data. To actually extract the textual data, you must call the selector ``extract()`` method, as follows:: @@ -126,12 +125,12 @@ method, as follows:: >>> hxs.select('//title/text()').extract() [u'Example website'] -Now notice that CSS selectors can't select the text nodes. There are some -methods that allow enhancing CSS selectors, such as ``text`` and ``get``:: +Now notice that CSS selectors can select text or attribute nodes using CSS3 +pseudo-elements:: - >>> hcs.select('title').text() + >>> hcs.select('title::text') [] - >>> hcs.select('title').text().extract() + >>> hcs.select('title::text').extract() [u'Example website'] Now we're going to get the base URL and some image links:: @@ -139,7 +138,7 @@ Now we're going to get the base URL and some image links:: >>> hxs.select('//base/@href').extract() [u'http://example.com/'] - >>> hcs.select('base').get('href') + >>> hcs.select('base::attr(href)').extract() [u'http://example.com/'] >>> hxs.select('//a[contains(@href, "image")]/@href').extract() @@ -149,7 +148,7 @@ Now we're going to get the base URL and some image links:: u'image4.html', u'image5.html'] - >>> hcs.select('a[href*=image]').get('href').extract() + >>> hcs.select('a[href*=image]::attr(href)').extract() [u'image1.html', u'image2.html', u'image3.html', @@ -163,7 +162,7 @@ Now we're going to get the base URL and some image links:: u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> hcs.select('a[href*=image] img').get('src').extract() + >>> hcs.select('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -197,28 +196,13 @@ Here's an example:: Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] -The CSSSelectorList ``select`` method will accept CSS selectors, as expected, -but it also provides an ``xpath`` method that accepts XPath selectors to -augment the CSS selectors. Here's an example:: - - >>> links = hcs.select('a[href*=image]') - >>> for index, link in enumerate(links): - args = (index, link.get('href').extract(), link.xpath('img/@src').extract()) - print 'Link number %d points to url %s and image %s' % args - - Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] - Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg'] - Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg'] - Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] - Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] - Using selectors with regular expressions ---------------------------------------- Selectors (both CSS and XPath) also have a ``re()`` method for extracting data using regular expressions. However, unlike using the ``select()`` method, the ``re()`` method does not return a list of -:class:`~scrapy.selector.XPathSelector` objects, so you can't construct nested +:class:`Selector` objects, so you can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code @@ -271,16 +255,71 @@ XPath specification. .. _topics-selectors-ref: Built-in Selectors reference -================================== +============================ .. module:: scrapy.selector :synopsis: Selectors classes There are four types of selectors bundled with Scrapy: :class:`HtmlXPathSelector` and :class:`XmlXPathSelector`, -:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. All of them implement the -same :class:`XPathSelector` interface. The only differences are the selector -syntax and whether it is used to process HTML data or XML data. +:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. + +All of them implement the same :class:`XPathSelector` interface. The only +differences are the selector syntax and whether it is used to process HTML data +or XML data. + +Selector interface +------------------ + +.. class:: Selector(response) + + An instance implementing :class:`Selector` interface is a wrapper over + ``response`` to select certain parts of its content. + + ``response`` is a :class:`~scrapy.http.Response` object that will be used + for selecting and extracting data. + + .. method:: select(query) + + Find nodes matching the selection query and return the result as a + :class:`SelectorList` instance with all elements flattened. List + elements must implement :class:`Selector` interface too. + + .. method:: extract() + + Serialize and return the matched nodes as a list of unicode strings + + .. method:: __nonzero__() + + Returns ``True`` if there is any real content selected or ``False`` + otherwise. In other words, the boolean value of a :class:`Selector` is + given by the contents it selects. + + +SelectorList objects +-------------------- + +.. class:: SelectorList + + The :class:`SelectorList` class is subclass of the builtin ``list`` + class, which provides a few additional methods. + + .. method:: select(query) + + Call the ``select()`` method for each element in this list and return + their results flattened as another :class:`SelectorList`. + + ``query`` is the same argument as the one in :meth:`Selector.select` + + .. method:: extract() + + Call the ``extract()`` method for each element is this list and return + their results flattened, as a list of unicode strings. + + .. method:: __nonzero__() + + returns True if the list is not empty, False otherwise. + .. _topics-xpath-selectors-ref: @@ -289,16 +328,18 @@ XPathSelector objects .. class:: XPathSelector(response) - A :class:`XPathSelector` object is a wrapper over response to select - certain parts of its content. + :class:`Selector` interface implementation that uses `XPath`_ query language + to select content on ``response`` ``response`` is a :class:`~scrapy.http.Response` object that will be used for selecting and extracting data. + In the background, XPath selectors are powered by `lxml`_ library + .. method:: select(xpath) Apply the given XPath relative to this XPathSelector and return a list - of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList`) + of :class:`XPathSelector` objects (ie. a :class:`SelectorList`) with the result. ``xpath`` is a string containing the XPath to apply @@ -327,50 +368,6 @@ XPathSelector objects Remove all namespaces, allowing to traverse the document using namespace-less xpaths. See example below. - .. method:: __nonzero__() - - Returns ``True`` if there is any real content selected by this - :class:`XPathSelector` or ``False`` otherwise. In other words, the - boolean value of an XPathSelector is given by the contents it selects. - -XPathSelectorList objects -------------------------- - -.. class:: XPathSelectorList - - The :class:`XPathSelectorList` class is subclass of the builtin ``list`` - class, which provides a few additional methods. - - .. method:: select(xpath) - - Call the :meth:`XPathSelector.select` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a new :class:`XPathSelectorList`. - - ``xpath`` is the same argument as the one in - :meth:`XPathSelector.select` - - .. method:: re(regex) - - Call the :meth:`XPathSelector.re` method for all :class:`XPathSelector` - objects in this list and return their results flattened, as a list of - unicode strings. - - ``regex`` is the same argument as the one in :meth:`XPathSelector.re` - - .. method:: extract() - - Call the :meth:`XPathSelector.extract` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a list of unicode strings. - - .. method:: extract_unquoted() - - Call the :meth:`XPathSelector.extract_unoquoted` method for all - :class:`XPathSelector` objects in this list and return their results - flattened, as a list of unicode strings. This method should not be - applied to all kinds of XPathSelectors. For more info see - :meth:`XPathSelector.extract_unoquoted`. HtmlXPathSelector objects ------------------------- @@ -378,11 +375,9 @@ HtmlXPathSelector objects .. class:: HtmlXPathSelector(response) A subclass of :class:`XPathSelector` for working with HTML content. It uses - the `libxml2`_ HTML parser. See the :class:`XPathSelector` API for more + the `lxml`_ HTML parser. See the :class:`XPathSelector` API for more info. -.. _libxml2: http://xmlsoft.org/ - HtmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -424,7 +419,7 @@ XmlXPathSelector objects .. class:: XmlXPathSelector(response) A subclass of :class:`XPathSelector` for working with XML content. It uses - the `libxml2`_ XML parser. See the :class:`XPathSelector` API for more info. + the `lxml`_ XML parser. See the :class:`XPathSelector` API for more info. XmlXPathSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -482,11 +477,11 @@ If you wonder why the namespace removal procedure is not always called, instead of having to call it manually. This is because of two reasons which, in order of relevance, are: -1. removing namespaces requires to iterate and modify all nodes in the +1. Removing namespaces requires to iterate and modify all nodes in the document, which is a reasonably expensive operation to performs for all documents crawled by Scrapy -2. there could be some cases where using namespaces is actually required, in +2. There could be some cases where using namespaces is actually required, in case some element names clash between namespaces. These cases are very rare though. @@ -495,144 +490,80 @@ of relevance, are: .. _topics-css-selectors-ref: CSSSelector objects --------------------- +------------------- -.. class:: CSSSelectorMixin(object) +.. class:: CSSSelector(response) - A :class:`CSSSelectorMixin` object is a mixin for either - :class:`XmlXPathSelector` or :class:`HtmlXPathSelector` to select element - nodes using CSS selectors syntax. As a mixin, it is not meant to be used on - its own, but as a secondary parent class. See :class:`XmlCSSSelector` and - :class:`HtmlCSSSelector` for implementations. + :class:`Selector` interface implementation that uses `CSS`_ query language + to select content on ``response`` + + ``response`` is a :class:`~scrapy.http.Response` object that will be used + for selecting and extracting data. + + In the background, CSS selectors are translated into XPath selectors using + `cssselect`_ library and run using :class:`XPathSelector` .. method:: select(css) - Apply the given CSS selector relative to this CSSSelectorMixin and - return a list of :class:`CSSSelectorMixin` objects (ie. a - :class:`CSSSelectorList`) with the result. + Apply the given CSS selector relative to this CSSSelector and return a + :class:`SelectorList` instance. ``css`` is a string containing the CSS selector to apply. - .. method:: xpath(xpath) - - Apply the given XPath relative to this CSSSelectorMixin and return a list - of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) - with the result. - - ``xpath`` is a string containing the XPath to apply. - - .. method:: get(attr) - - Get the attribute relative to this CSSSelectorMixin and return a list - of :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) - with the result (usually with one element only). - - ``attr`` is a string containing the attribute name to get. - - .. method:: text(all=False) - - Get the children text nodes relative to this CSSSelectorMixin or, if - ``all`` is True, a string node concatenating all of the descendant text - nodes relative to this CSSSelectorMixin, and return a list of - :class:`CSSSelectorMixin` objects (ie. a :class:`CSSSelectorList`) with - the result. - - ``all`` is a boolean to either select children text nodes (False) or - select a string node concatenating all of the descendant text nodes. - -CSSSelectorList objects ------------------------ - -.. class:: CSSSelectorList - - The :class:`CSSSelectorList` class is subclass of :class:`XPathSelectorList` - which overrides and adds methods to match those of - :class:`CSSSelectorMixin`. - - .. method:: xpath(xpath) - - Call the :meth:`CSSSelectorMixin.xpath` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``xpath`` is the same argument as the one in - :meth:`CSSSelectorMixin.xpath` - - .. method:: get(attr) - - Call the :meth:`CSSSelectorMixin.get` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``attr`` is the same argument as the one in :meth:`CSSSelectorMixin.get` - - .. method:: text(all=False) - - Call the :meth:`CSSSelectorMixin.text` method for all - :class:`CSSSelectorMixin` objects in this list and return their results - flattened, as a new :class:`CSSSelectorList`. - - ``all`` is the same argument as the one in :meth:`CSSSelectorMixin.text` - HtmlCSSSelector objects ----------------------- .. class:: HtmlCSSSelector(response) - A subclass of :class:`CSSSelectorMixin` and :class:`HtmlXPathSelector` for - working with HTML content using CSS selectors. + A specialized class for working with HTML content using `CSS`_ selectors. HtmlCSSSelector examples ~~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`HtmlCSSSelector` examples to illustrate several concepts. In all cases, we assume there is already an :class:`HtmlCSSSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: +instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: x = HtmlCSSSelector(html_response) 1. Select all ``

`` elements from a HTML response body, returning a list of - :class:`HtmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + :class:`HtmlCSSSelector` objects:: x.select("h1") 2. Extract the text of all ``

`` elements from a HTML response body, returning a list of unicode strings:: - x.select("h1").extract() # this includes the h1 tag - x.select("h1").text().extract() # this excludes the h1 tag + x.select("h1").extract() # Includes the h1 tag + x.select("h1::text").extract() # Only text inside the h1 tag 3. Iterate over all ``

`` tags and print their class attribute:: for node in x.select("p"): - ... print node.get("class").extract() + ... print node.select("::attr(class)").extract() XmlCSSSelector objects ---------------------- .. class:: XmlCSSSelector(response) - A subclass of :class:`CSSSelectorMixin` and :class:`XmlXPathSelector` for - working with XML content using CSS selectors. + A specialized class for working with XML content using `CSS`_ selectors. XmlCSSSelector examples ~~~~~~~~~~~~~~~~~~~~~~~ Here's a couple of :class:`XmlCSSSelector` examples to illustrate several concepts. In both cases we assume there is already an :class:`XmlCSSSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: +instantiated with a :class:`~scrapy.http.XmlResponse` object like this:: x = XmlCSSSelector(xml_response) 1. Select all ```` elements from a XML response body, returning a list - of :class:`XmlCSSSelector` objects (ie. a :class:`CSSSelectorList` object):: + of :class:`XmlCSSSelector` objects:: x.select("product") -2. Extract all prices from a `Google Base XML feed`_ which requires registering - a namespace:: - - x.register_namespace("g", "http://base.google.com/ns/1.0") - x.xpath("//g:price").extract() +Note that querying xml namespaces with CSS selectors doesn't work, if you are +interesting in this feature consider contributing to `cssselect`_ project. .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py index ea634db1f..774be1155 100644 --- a/scrapy/selector/list.py +++ b/scrapy/selector/list.py @@ -16,6 +16,7 @@ class SelectorList(list): def extract(self): return [x.extract() for x in self] + @deprecated(use_instead='SelectorList.extract') def extract_unquoted(self): return [x.extract_unquoted() for x in self] From bf37f7857210cf225f4de7df31560105a26d2d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 10 Oct 2013 19:02:55 -0200 Subject: [PATCH 07/19] Drop libxml2 selectors backend --- docs/news.rst | 2 + docs/topics/extensions.rst | 1 - docs/topics/selectors.rst | 1 - scrapy/__init__.py | 7 -- scrapy/commands/version.py | 11 +-- scrapy/contrib/memdebug.py | 14 +-- scrapy/selector/__init__.py | 26 +----- scrapy/selector/libxml2document.py | 82 ------------------ scrapy/selector/libxml2sel.py | 118 -------------------------- scrapy/tests/test_libxml2.py | 20 ----- scrapy/tests/test_selector.py | 25 +----- scrapy/tests/test_selector_libxml2.py | 98 --------------------- scrapy/tests/test_selector_lxml.py | 4 +- scrapy/utils/test.py | 24 ------ 14 files changed, 12 insertions(+), 421 deletions(-) delete mode 100644 scrapy/selector/libxml2document.py delete mode 100644 scrapy/selector/libxml2sel.py delete mode 100644 scrapy/tests/test_libxml2.py delete mode 100644 scrapy/tests/test_selector_libxml2.py diff --git a/docs/news.rst b/docs/news.rst index 8d6022fe7..dd4f571b6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,6 +9,8 @@ Release notes - Request/Response url/body attributes are now immutable (modifying them had been deprecated for a long time) - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) +- Dropped libxml2 selectors backend +- Dropped support for multiple selectors backends, sticking to lxml only 0.18.4 (released 2013-10-10) ---------------------------- diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 9a4c0bf2c..eb944fa34 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -248,7 +248,6 @@ Memory debugger extension An extension for debugging memory usage. It collects information about: * objects uncollected by the Python garbage collector -* libxml2 memory leaks * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cd4d825d9..554f16eb6 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -40,7 +40,6 @@ reference ` and :ref:`CSS selector reference .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://codespeak.net/lxml/ .. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html -.. _libxml2: http://xmlsoft.org/ .. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors diff --git a/scrapy/__init__.py b/scrapy/__init__.py index e32618ebb..fef482d52 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -30,13 +30,6 @@ except ImportError: else: optional_features.add('boto') -try: - import libxml2 -except ImportError: - pass -else: - optional_features.add('libxml2') - try: import django except ImportError: diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 27e12046a..36210a04b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -6,6 +6,7 @@ import twisted import scrapy from scrapy.command import ScrapyCommand + class Command(ScrapyCommand): def syntax(self): @@ -21,13 +22,9 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - try: - import lxml.etree - except ImportError: - lxml_version = libxml2_version = "(lxml not available)" - else: - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + import lxml.etree + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) print "Scrapy : %s" % scrapy.__version__ print "lxml : %s" % lxml_version print "libxml2 : %s" % libxml2_version diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index 359c7ec8a..2d9249f96 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -10,14 +10,10 @@ from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs + class MemoryDebugger(object): def __init__(self, stats): - try: - import libxml2 - self.libxml2 = libxml2 - except ImportError: - self.libxml2 = None self.stats = stats @classmethod @@ -25,18 +21,10 @@ class MemoryDebugger(object): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured o = cls(crawler.stats) - crawler.signals.connect(o.engine_started, signals.engine_started) crawler.signals.connect(o.engine_stopped, signals.engine_stopped) return o - def engine_started(self): - if self.libxml2: - self.libxml2.debugMemory(1) - def engine_stopped(self): - if self.libxml2: - self.libxml2.cleanupParser() - self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) for cls, wdict in live_refs.iteritems(): diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 2b855d147..7dd420d1d 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,29 +1,7 @@ """ -XPath selectors - -To select the backend explicitly use the SCRAPY_SELECTORS_BACKEND environment -variable. - -Two backends are currently available: lxml (default) and libxml2. - +Selectors """ -import os - - -backend = os.environ.get('SCRAPY_SELECTORS_BACKEND') -if backend == 'libxml2': - from scrapy.selector.libxml2sel import * -elif backend == 'lxml': - from scrapy.selector.lxmlsel import * -else: - try: - import lxml - except ImportError: - import libxml2 - from scrapy.selector.libxml2sel import * - else: - from scrapy.selector.lxmlsel import * - +from scrapy.selector.lxmlsel import * from scrapy.selector.csssel import * from scrapy.selector.list import SelectorList diff --git a/scrapy/selector/libxml2document.py b/scrapy/selector/libxml2document.py deleted file mode 100644 index 070dd0c58..000000000 --- a/scrapy/selector/libxml2document.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -This module contains a simple class (Libxml2Document) which provides cache and -garbage collection to libxml2 documents (xmlDoc). -""" - -import weakref -from scrapy.utils.trackref import object_ref -from scrapy import optional_features - -if 'libxml2' in optional_features: - import libxml2 - xml_parser_options = libxml2.XML_PARSE_RECOVER + \ - libxml2.XML_PARSE_NOERROR + \ - libxml2.XML_PARSE_NOWARNING - - html_parser_options = libxml2.HTML_PARSE_RECOVER + \ - libxml2.HTML_PARSE_NOERROR + \ - libxml2.HTML_PARSE_NOWARNING - - -_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) -def _body_as_utf8(response): - if response.encoding in _UTF8_ENCODINGS: - return response.body - else: - return response.body_as_unicode().encode('utf-8') - - -def xmlDoc_from_html(response): - """Return libxml2 doc for HTMLs""" - utf8body = _body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ - html_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', html_parser_options) - return lxdoc - - -def xmlDoc_from_xml(response): - """Return libxml2 doc for XMLs""" - utf8body = _body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ - xml_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', xml_parser_options) - return lxdoc - - -class Libxml2Document(object_ref): - - cache = weakref.WeakKeyDictionary() - __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] - - def __new__(cls, response, factory=xmlDoc_from_html): - cache = cls.cache.setdefault(response, {}) - if factory not in cache: - obj = object_ref.__new__(cls) - obj.xmlDoc = factory(response) - obj.xpathContext = obj.xmlDoc.xpathNewContext() - cache[factory] = obj - return cache[factory] - - def __del__(self): - # we must call both cleanup functions, so we try/except all exceptions - # to make sure one doesn't prevent the other from being called - # this call sometimes raises a "NoneType is not callable" TypeError - # so the try/except block silences them - try: - self.xmlDoc.freeDoc() - except: - pass - try: - self.xpathContext.xpathFreeContext() - except: - pass - - def __str__(self): - return "" % self.xmlDoc.name diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py deleted file mode 100644 index 2f5d72ea1..000000000 --- a/scrapy/selector/libxml2sel.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -XPath selectors based on libxml2 -""" - -from scrapy import optional_features -if 'libxml2' in optional_features: - import libxml2 - -from scrapy.http import TextResponse -from scrapy.utils.python import unicode_to_str -from scrapy.utils.misc import extract_regex -from scrapy.utils.trackref import object_ref -from scrapy.utils.decorator import deprecated -from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml -from .list import SelectorList - - -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] - - -class XPathSelector(object_ref): - - __slots__ = ['doc', 'xmlNode', 'expr', '__weakref__'] - - def __init__(self, response=None, text=None, node=None, parent=None, expr=None): - if parent is not None: - self.doc = parent.doc - self.xmlNode = node - elif response: - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - elif text: - response = TextResponse(url='about:blank', \ - body=unicode_to_str(text, 'utf-8'), encoding='utf-8') - self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) - self.xmlNode = self.doc.xmlDoc - self.expr = expr - - def select(self, xpath): - if hasattr(self.xmlNode, 'xpathEval'): - self.doc.xpathContext.setContextNode(self.xmlNode) - xpath = unicode_to_str(xpath, 'utf-8') - try: - xpath_result = self.doc.xpathContext.xpathEval(xpath) - except libxml2.xpathError: - raise ValueError("Invalid XPath: %s" % xpath) - if hasattr(xpath_result, '__iter__'): - return SelectorList([self.__class__(node=node, parent=self, \ - expr=xpath) for node in xpath_result]) - else: - return SelectorList([self.__class__(node=xpath_result, \ - parent=self, expr=xpath)]) - else: - return SelectorList([]) - - def re(self, regex): - return extract_regex(regex, self.extract()) - - def extract(self): - if isinstance(self.xmlNode, basestring): - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - elif hasattr(self.xmlNode, 'serialize'): - if isinstance(self.xmlNode, libxml2.xmlDoc): - data = self.xmlNode.getRootElement().serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - elif isinstance(self.xmlNode, libxml2.xmlAttr): - # serialization doesn't work sometimes for xmlAttr types - text = unicode(self.xmlNode.content, 'utf-8', errors='ignore') - else: - data = self.xmlNode.serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - else: - try: - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - except TypeError: # catched when self.xmlNode is a float - see tests - text = unicode(self.xmlNode) - return text - - def extract_unquoted(self): - """Get unescaped contents from the text node (no entities, no CDATA)""" - # TODO: this function should be deprecated. but what would be use instead? - if self.select('self::text()'): - return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') - else: - return u'' - - def register_namespace(self, prefix, uri): - self.doc.xpathContext.xpathRegisterNs(prefix, uri) - - def _get_libxml2_doc(self, response): - return xmlDoc_from_html(response) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) - - __repr__ = __str__ - - @deprecated(use_instead='XPathSelector.select') - def __call__(self, xpath): - return self.select(xpath) - - @deprecated(use_instead='XPathSelector.select') - def x(self, xpath): - return self.select(xpath) - - -class XmlXPathSelector(XPathSelector): - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_xml) - - -class HtmlXPathSelector(XPathSelector): - __slots__ = () - _get_libxml2_doc = staticmethod(xmlDoc_from_html) diff --git a/scrapy/tests/test_libxml2.py b/scrapy/tests/test_libxml2.py deleted file mode 100644 index e042de526..000000000 --- a/scrapy/tests/test_libxml2.py +++ /dev/null @@ -1,20 +0,0 @@ -from twisted.trial import unittest - -from scrapy.utils.test import libxml2debug -from scrapy import optional_features - - -class Libxml2Test(unittest.TestCase): - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_libxml2_bug_2_6_27(self): - # this test will fail in version 2.6.27 but passes on 2.6.29+ - import libxml2 - html = "123" - node = libxml2.htmlParseDoc(html, 'utf-8') - result = [str(r) for r in node.xpathEval('//text()')] - self.assertEquals(result, ['1', '2', '3']) - node.freeDoc() - diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 284c28db6..fc732a694 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -10,7 +10,7 @@ from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector -from scrapy.utils.test import libxml2debug + class XPathSelectorTestCase(unittest.TestCase): @@ -18,7 +18,6 @@ class XPathSelectorTestCase(unittest.TestCase): hxs_cls = HtmlXPathSelector xxs_cls = XmlXPathSelector - @libxml2debug def test_selector_simple(self): """Simple selector tests""" body = "

" @@ -43,23 +42,20 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) - @libxml2debug def test_selector_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8') xpath = self.hxs_cls(response) self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1']) - @libxml2debug def test_selector_same_type(self): """Test XPathSelector returning the same type in x() method""" text = '

test

' assert isinstance(self.xxs_cls(text=text).select("//p")[0], self.xxs_cls) - assert isinstance(self.hxs_cls(text=text).select("//p")[0], + assert isinstance(self.hxs_cls(text=text).select("//p")[0], self.hxs_cls) - @libxml2debug def test_selector_boolean_result(self): body = "

" response = TextResponse(url="http://example.com", body=body) @@ -67,7 +63,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1']) self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0']) - @libxml2debug def test_selector_xml_html(self): """Test that XML and HTML XPathSelector's behave differently""" @@ -80,7 +75,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(self.hxs_cls(text=text).select("//div").extract(), [u'

Hello

']) - @libxml2debug def test_selector_nested(self): """Nested selector tests""" body = """ @@ -109,13 +103,11 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(divtwo.select("./li").extract(), []) - @libxml2debug def test_dont_strip(self): hxs = self.hxs_cls(text='
fff: zzz
') self.assertEqual(hxs.select("//text()").extract(), [u'fff: ', u'zzz']) - @libxml2debug def test_selector_namespaces_simple(self): body = """ @@ -131,7 +123,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//somens:a/text()").extract(), [u'take this']) - @libxml2debug def test_selector_namespaces_multiple(self): body = """ Name: Mary
    @@ -177,14 +167,12 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), ["10", "20"]) - @libxml2debug def test_selector_re_intl(self): body = """
    Evento: cumplea\xc3\xb1os
    """ response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8') x = self.hxs_cls(response) self.assertEqual(x.select("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) - @libxml2debug def test_selector_over_text(self): hxs = self.hxs_cls(text='lala') self.assertEqual(hxs.extract(), @@ -199,7 +187,6 @@ class XPathSelectorTestCase(unittest.TestCase): [u'lala']) - @libxml2debug def test_selector_invalid_xpath(self): response = XmlResponse(url="http://example.com", body="") x = self.hxs_cls(response) @@ -213,7 +200,6 @@ class XPathSelectorTestCase(unittest.TestCase): else: raise AssertionError("A invalid XPath does not raise an exception") - @libxml2debug def test_http_header_encoding_precedence(self): # u'\xa3' = pound symbol in unicode # u'\xc2\xa3' = pound symbol in utf-8 @@ -233,14 +219,12 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x.select("//span[@id='blank']/text()").extract(), [u'\xa3']) - @libxml2debug def test_empty_bodies(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_null_bytes(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', \ @@ -249,7 +233,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence r1 = TextResponse('http://www.example.com', \ @@ -258,7 +241,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.hxs_cls(r1).select('//text()').extract() self.xxs_cls(r1).select('//text()').extract() - @libxml2debug def test_select_on_unevaluable_nodes(self): r = self.hxs_cls(text=u'some text') # Text node @@ -270,7 +252,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x1.extract(), [u'big']) self.assertEquals(x1.select('.//text()').extract(), []) - @libxml2debug def test_select_on_text_nodes(self): r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") @@ -279,7 +260,6 @@ class XPathSelectorTestCase(unittest.TestCase): x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x1.extract(), [u'Options:']) - @libxml2debug def test_nested_select_on_text_nodes(self): # FIXME: does not work with lxml backend [upstream] r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') @@ -289,7 +269,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEquals(x2.extract(), [u'Options:']) test_nested_select_on_text_nodes.skip = True - @libxml2debug def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" for cls in [self.xs_cls, self.hxs_cls, self.xxs_cls]: diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py deleted file mode 100644 index 5dabd2d01..000000000 --- a/scrapy/tests/test_selector_libxml2.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Selectors tests, specific for libxml2 backend -""" - -from twisted.trial import unittest -from scrapy import optional_features - - -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector -from scrapy.selector.libxml2document import Libxml2Document -from scrapy.utils.test import libxml2debug -from scrapy.tests import test_selector - - -class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase): - - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_null_bytes(self): - hxs = HtmlXPathSelector(text='la\x00la') - self.assertEqual(hxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='la\x00la') - self.assertEqual(xxs.extract(), - u'lala') - - @libxml2debug - def test_unquote(self): - xmldoc = '\n'.join(( - '', - ' lala', - ' ', - ' blabla&moreatestoh', - ' PPPPppp&la]]>', - ' ', - ' pff', - '')) - xxs = XmlXPathSelector(text=xmldoc) - - self.assertEqual(xxs.extract_unquoted(), u'') - - self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) - self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n pff\n']) - - self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) - self.assertEqual(xxs.select('//text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n blabla&more', - u'a', - u'test', - u'oh\n ', - u'lalalal&pppppPPPPppp&la', - u'\n ', - u'\n pff\n']) - - -class Libxml2DocumentTest(unittest.TestCase): - - skip = 'libxml2' not in optional_features - - @libxml2debug - def test_response_libxml2_caching(self): - r1 = HtmlResponse('http://www.example.com', body='') - r2 = r1.copy() - - doc1 = Libxml2Document(r1) - doc2 = Libxml2Document(r1) - doc3 = Libxml2Document(r2) - - # make sure it's cached - assert doc1 is doc2 - assert doc1.xmlDoc is doc2.xmlDoc - assert doc1 is not doc3 - assert doc1.xmlDoc is not doc3.xmlDoc - - # don't leave libxml2 documents in memory to avoid wrong libxml2 leaks reports - del doc1, doc2, doc3 - - @libxml2debug - def test_null_char(self): - # make sure bodies with null char ('\x00') don't raise a TypeError exception - self.body_content = 'test problematic \x00 body' - response = TextResponse('http://example.com/catalog/product/blabla-123', - headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) - Libxml2Document(response) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 352861ae6..7ea674d62 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -39,6 +39,7 @@ class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): xxs.remove_namespaces() self.assertEqual(len(xxs.select("//link/@type")), 2) + class Libxml2DocumentTest(unittest.TestCase): def test_caching(self): @@ -53,9 +54,6 @@ class Libxml2DocumentTest(unittest.TestCase): assert doc1 is doc2 assert doc1 is not doc3 - # don't leave documents in memory to avoid wrong libxml2 leaks reports - del doc1, doc2, doc3 - def test_null_char(self): # make sure bodies with null char ('\x00') don't raise a TypeError exception self.body_content = 'test problematic \x00 body' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index a931443a6..2695a2921 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -6,30 +6,6 @@ import os from twisted.trial.unittest import SkipTest -def libxml2debug(testfunction): - """Decorator for debugging libxml2 memory leaks inside a function. - - We've found libxml2 memory leaks are something very weird, and can happen - sometimes depending on the order where tests are run. So this decorator - enables libxml2 memory leaks debugging only when the environment variable - LIBXML2_DEBUGLEAKS is set. - - """ - try: - import libxml2 - except ImportError: - return testfunction - def newfunc(*args, **kwargs): - libxml2.debugMemory(1) - testfunction(*args, **kwargs) - libxml2.cleanupParser() - leaked_bytes = libxml2.debugMemory(0) - assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes - - if 'LIBXML2_DEBUGLEAKS' in os.environ: - return newfunc - else: - return testfunction def assert_aws_environ(): """Asserts the current environment is suitable for running AWS testsi. From c3d28cc4128d2eb387c5c89e36fb5f4261546bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 11 Oct 2013 18:06:27 -0200 Subject: [PATCH 08/19] working implementaion of unified api --- scrapy/selector/__init__.py | 13 +- scrapy/selector/csssel.py | 108 +--------------- scrapy/selector/csstranslator.py | 88 +++++++++++++ scrapy/selector/list.py | 25 ---- scrapy/selector/lxmlsel.py | 110 +++------------- scrapy/selector/unified.py | 164 ++++++++++++++++++++++++ scrapy/tests/test_selector_cssselect.py | 2 +- 7 files changed, 280 insertions(+), 230 deletions(-) create mode 100644 scrapy/selector/csstranslator.py delete mode 100644 scrapy/selector/list.py create mode 100644 scrapy/selector/unified.py diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 7dd420d1d..a1203f987 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,17 +1,6 @@ """ Selectors """ +from scrapy.selector.unified import * from scrapy.selector.lxmlsel import * from scrapy.selector.csssel import * -from scrapy.selector.list import SelectorList - - -class XPathSelectorList(SelectorList): - - def __init__(self, *a, **kw): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('XPathSelectorList is deprecated, use ' - 'scrapy.selector.SelectorList instead', - category=ScrapyDeprecationWarning, stacklevel=1) - super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index e8c5538aa..a2fc12e8d 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,110 +1,16 @@ -from cssselect import GenericTranslator, HTMLTranslator -from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError -from cssselect.parser import FunctionalPseudoElement -from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector +from .unified import Selector -class ScrapyXPathExpr(XPathExpr): +class CSSSelector(Selector): - textnode = False - attribute = None - - @classmethod - def from_xpath(cls, xpath, textnode=False, attribute=None): - x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) - x.textnode = textnode - x.attribute = attribute - return x - - def __str__(self): - path = super(ScrapyXPathExpr, self).__str__() - if self.textnode: - if path == '*': - path = 'text()' - elif path.endswith('::*/*'): - path = path[:-3] + 'text()' - else: - path += '/text()' - - if self.attribute is not None: - if path.endswith('::*/*'): - path = path[:-2] - path += '/@%s' % self.attribute - - return path - - def join(self, combiner, other): - super(ScrapyXPathExpr, self).join(combiner, other) - self.textnode = other.textnode - self.attribute = other.attribute - return self - - -class TranslatorMixin(object): - - def xpath_element(self, selector): - xpath = super(TranslatorMixin, self).xpath_element(selector) - return ScrapyXPathExpr.from_xpath(xpath) - - def xpath_pseudo_element(self, xpath, pseudo_element): - if isinstance(pseudo_element, FunctionalPseudoElement): - method = 'xpath_%s_functional_pseudo_element' % ( - pseudo_element.name.replace('-', '_')) - method = _unicode_safe_getattr(self, method, None) - if not method: - raise ExpressionError( - "The functional pseudo-element ::%s() is unknown" - % pseudo_element.name) - xpath = method(xpath, pseudo_element) - else: - method = 'xpath_%s_simple_pseudo_element' % ( - pseudo_element.replace('-', '_')) - method = _unicode_safe_getattr(self, method, None) - if not method: - raise ExpressionError( - "The pseudo-element ::%s is unknown" - % pseudo_element) - xpath = method(xpath) - return xpath - - def xpath_attr_functional_pseudo_element(self, xpath, function): - if function.argument_types() not in (['STRING'], ['IDENT']): - raise ExpressionError( - "Expected a single string or ident for ::attr(), got %r" - % function.arguments) - return ScrapyXPathExpr.from_xpath(xpath, - attribute=function.arguments[0].value) - - def xpath_text_simple_pseudo_element(self, xpath): - """Support selecting text nodes using ::text pseudo-element""" - return ScrapyXPathExpr.from_xpath(xpath, textnode=True) - - -class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): - pass - - -class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator): - pass - - -class CSSSelectorMixin(object): + default_contenttype = 'html' def select(self, css): - xpath = self._css2xpath(css) - return super(CSSSelectorMixin, self).select(xpath) - - def _css2xpath(self, css): - return self.translator.css_to_xpath(css) + return self.css(css) -class CSSSelector(CSSSelectorMixin, XPathSelector): - translator = ScrapyHTMLTranslator() +HtmlCSSSelector = CSSSelector -class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector): - translator = ScrapyHTMLTranslator() - - -class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): - translator = ScrapyGenericTranslator() +class XmlCSSSelector(CSSSelector): + default_contenttype = 'xml' diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py new file mode 100644 index 000000000..7482837a0 --- /dev/null +++ b/scrapy/selector/csstranslator.py @@ -0,0 +1,88 @@ +from cssselect import GenericTranslator, HTMLTranslator +from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError +from cssselect.parser import FunctionalPseudoElement + + +class ScrapyXPathExpr(XPathExpr): + + textnode = False + attribute = None + + @classmethod + def from_xpath(cls, xpath, textnode=False, attribute=None): + x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) + x.textnode = textnode + x.attribute = attribute + return x + + def __str__(self): + path = super(ScrapyXPathExpr, self).__str__() + if self.textnode: + if path == '*': + path = 'text()' + elif path.endswith('::*/*'): + path = path[:-3] + 'text()' + else: + path += '/text()' + + if self.attribute is not None: + if path.endswith('::*/*'): + path = path[:-2] + path += '/@%s' % self.attribute + + return path + + def join(self, combiner, other): + super(ScrapyXPathExpr, self).join(combiner, other) + self.textnode = other.textnode + self.attribute = other.attribute + return self + + +class TranslatorMixin(object): + + def xpath_element(self, selector): + xpath = super(TranslatorMixin, self).xpath_element(selector) + return ScrapyXPathExpr.from_xpath(xpath) + + def xpath_pseudo_element(self, xpath, pseudo_element): + if isinstance(pseudo_element, FunctionalPseudoElement): + method = 'xpath_%s_functional_pseudo_element' % ( + pseudo_element.name.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The functional pseudo-element ::%s() is unknown" + % pseudo_element.name) + xpath = method(xpath, pseudo_element) + else: + method = 'xpath_%s_simple_pseudo_element' % ( + pseudo_element.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The pseudo-element ::%s is unknown" + % pseudo_element) + xpath = method(xpath) + return xpath + + def xpath_attr_functional_pseudo_element(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for ::attr(), got %r" + % function.arguments) + return ScrapyXPathExpr.from_xpath(xpath, + attribute=function.arguments[0].value) + + def xpath_text_simple_pseudo_element(self, xpath): + """Support selecting text nodes using ::text pseudo-element""" + return ScrapyXPathExpr.from_xpath(xpath, textnode=True) + + +class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): + pass + + +class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator): + pass + diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py deleted file mode 100644 index 774be1155..000000000 --- a/scrapy/selector/list.py +++ /dev/null @@ -1,25 +0,0 @@ -from scrapy.utils.python import flatten -from scrapy.utils.decorator import deprecated - - -class SelectorList(list): - - def __getslice__(self, i, j): - return self.__class__(list.__getslice__(self, i, j)) - - def select(self, xpath): - return self.__class__(flatten([x.select(xpath) for x in self])) - - def re(self, regex): - return flatten([x.re(regex) for x in self]) - - def extract(self): - return [x.extract() for x in self] - - @deprecated(use_instead='SelectorList.extract') - def extract_unquoted(self): - return [x.extract_unquoted() for x in self] - - @deprecated(use_instead='SelectorList.select') - def x(self, xpath): - return self.select(xpath) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 18b3e5044..b37b5d74c 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -1,106 +1,34 @@ """ XPath selectors based on lxml """ - -from lxml import etree - -from scrapy.utils.misc import extract_regex -from scrapy.utils.trackref import object_ref -from scrapy.utils.python import unicode_to_str -from scrapy.utils.decorator import deprecated -from scrapy.http import TextResponse -from .lxmldocument import LxmlDocument -from .list import SelectorList +from .unified import Selector, SelectorList -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', + 'XPathSelectorList'] -class XPathSelector(object_ref): - - __slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__'] - _parser = etree.HTMLParser - _tostring_method = 'html' - - def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None): - if text is not None: - response = TextResponse(url='about:blank', encoding='utf-8', - body=unicode_to_str(text, 'utf-8')) - if response is not None: - _root = LxmlDocument(response, self._parser) - - self.namespaces = namespaces - self.response = response - self._root = _root - self._expr = _expr - - def select(self, xpath): - try: - xpathev = self._root.xpath - except AttributeError: - return SelectorList([]) - - try: - result = xpathev(xpath, namespaces=self.namespaces) - except etree.XPathError: - raise ValueError("Invalid XPath: %s" % xpath) - - if type(result) is not list: - result = [result] - - result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces) - for x in result] - return SelectorList(result) - - def re(self, regex): - return extract_regex(regex, self.extract()) - - def extract(self): - try: - return etree.tostring(self._root, method=self._tostring_method, \ - encoding=unicode, with_tail=False) - except (AttributeError, TypeError): - if self._root is True: - return u'1' - elif self._root is False: - return u'0' - else: - return unicode(self._root) - - def register_namespace(self, prefix, uri): - if self.namespaces is None: - self.namespaces = {} - self.namespaces[prefix] = uri - - def remove_namespaces(self): - for el in self._root.iter('*'): - if el.tag.startswith('{'): - el.tag = el.tag.split('}', 1)[1] - # loop on element attributes also - for an in el.attrib.keys(): - if an.startswith('{'): - el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) - __repr__ = __str__ - - @deprecated(use_instead='XPathSelector.extract') - def extract_unquoted(self): - return self.extract() +class XPathSelector(Selector): + __slots__ = () + default_contenttype = 'xml' class XmlXPathSelector(XPathSelector): __slots__ = () - _parser = etree.XMLParser - _tostring_method = 'xml' + default_contenttype = 'xml' class HtmlXPathSelector(XPathSelector): __slots__ = () - _parser = etree.HTMLParser - _tostring_method = 'html' + default_contenttype = 'html' + + +class XPathSelectorList(SelectorList): + + def __init__(self, *a, **kw): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('XPathSelectorList is deprecated, use ' + 'scrapy.selector.SelectorList instead', + category=ScrapyDeprecationWarning, stacklevel=1) + super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py new file mode 100644 index 000000000..12d0188ab --- /dev/null +++ b/scrapy/selector/unified.py @@ -0,0 +1,164 @@ +""" +XPath selectors based on lxml +""" + +from lxml import etree + +from scrapy.utils.misc import extract_regex +from scrapy.utils.trackref import object_ref +from scrapy.utils.python import unicode_to_str, flatten +from scrapy.utils.decorator import deprecated +from scrapy.http import HtmlResponse, XmlResponse +from .lxmldocument import LxmlDocument +from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator + + +__all__ = ['Selector', 'SelectorList'] + +_ctgroup = { + 'html': {'_parser': etree.HTMLParser, + '_csstranslator': ScrapyHTMLTranslator(), + '_tostring_method': 'html'}, + 'xml': {'_parser': etree.XMLParser, + '_csstranslator': ScrapyGenericTranslator(), + '_tostring_method': 'xml'}, +} + + +def _ct(response, ct): + if ct is None: + return 'xml' if isinstance(response, XmlResponse) else 'html' + elif ct in ('xml', 'html'): + return ct + else: + raise ValueError('Invalid contenttype: %s' % ct) + + +def _response_from_text(text, ct): + rt = XmlResponse if ct == 'xml' else HtmlResponse + return rt(url='about:blank', encoding='utf-8', + body=unicode_to_str(text, 'utf-8')) + + +class Selector(object_ref): + + __slots__ = ['response', 'text', 'namespaces', 'contenttype', '_expr', '_root', + '__weakref__', '_parser', '_csstranslator', '_tostring_method'] + + default_contenttype = None + + def __init__(self, response=None, text=None, namespaces=None, contenttype=None, + _root=None, _expr=None): + + self.contenttype = ct = self.default_contenttype or _ct(contenttype) + self._parser = _ctgroup[ct]['_parser'] + self._csstranslator = _ctgroup[ct]['_csstranslator'] + self._tostring_method = _ctgroup[ct]['_tostring_method'] + + if text is not None: + response = _response_from_text(text, ct) + + if response is not None: + _root = LxmlDocument(response, self._parser) + + self.response = response + self.namespaces = namespaces + self._root = _root + self._expr = _expr + + def xpath(self, query): + try: + xpathev = self._root.xpath + except AttributeError: + return SelectorList([]) + + try: + result = xpathev(query, namespaces=self.namespaces) + except etree.XPathError: + raise ValueError("Invalid XPath: %s" % query) + + if type(result) is not list: + result = [result] + + result = [self.__class__(_root=x, _expr=query, + namespaces=self.namespaces, + contenttype=self.contenttype) + for x in result] + return SelectorList(result) + + def css(self, query): + return self.xpath(self._css2xpath(query)) + + def _css2xpath(self, query): + return self._csstranslator.css_to_xpath(query) + + def re(self, regex): + return extract_regex(regex, self.extract()) + + def extract(self): + try: + return etree.tostring(self._root, + method=self._tostring_method, + encoding=unicode, + with_tail=False) + except (AttributeError, TypeError): + if self._root is True: + return u'1' + elif self._root is False: + return u'0' + else: + return unicode(self._root) + + def register_namespace(self, prefix, uri): + if self.namespaces is None: + self.namespaces = {} + self.namespaces[prefix] = uri + + def remove_namespaces(self): + for el in self._root.iter('*'): + if el.tag.startswith('{'): + el.tag = el.tag.split('}', 1)[1] + # loop on element attributes also + for an in el.attrib.keys(): + if an.startswith('{'): + el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an) + + def __nonzero__(self): + return bool(self.extract()) + + def __str__(self): + data = repr(self.extract()[:40]) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) + __repr__ = __str__ + + # Deprecated api + @deprecated(use_instead='.xpath()') + def select(self, xpath): + return self.xpath(xpath) + + @deprecated(use_instead='.extract()') + def extract_unquoted(self): + return self.extract() + + +class SelectorList(list): + + def __getslice__(self, i, j): + return self.__class__(list.__getslice__(self, i, j)) + + def select(self, xpath): + return self.__class__(flatten([x.select(xpath) for x in self])) + + def re(self, regex): + return flatten([x.re(regex) for x in self]) + + def extract(self): + return [x.extract() for x in self] + + @deprecated(use_instead='SelectorList.extract') + def extract_unquoted(self): + return [x.extract_unquoted() for x in self] + + @deprecated(use_instead='SelectorList.select') + def x(self, xpath): + return self.select(xpath) diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index 4b69dff15..399820240 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -4,7 +4,7 @@ Selector tests for cssselect backend from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector -from scrapy.selector.csssel import ScrapyHTMLTranslator +from scrapy.selector.csstranslator import ScrapyHTMLTranslator from cssselect.parser import SelectorSyntaxError from cssselect.xpath import ExpressionError From 681af6b2581b97f222eb95e7d4699247d63588f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 11 Oct 2013 18:22:03 -0200 Subject: [PATCH 09/19] Remove CSS*Selector classes and port its tests --- scrapy/selector/__init__.py | 1 - scrapy/selector/csssel.py | 16 ---------------- scrapy/selector/unified.py | 18 +++++++++++++----- scrapy/tests/test_selector_cssselect.py | 18 +++++++++--------- 4 files changed, 22 insertions(+), 31 deletions(-) delete mode 100644 scrapy/selector/csssel.py diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index a1203f987..bfbde4de9 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -3,4 +3,3 @@ Selectors """ from scrapy.selector.unified import * from scrapy.selector.lxmlsel import * -from scrapy.selector.csssel import * diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py deleted file mode 100644 index a2fc12e8d..000000000 --- a/scrapy/selector/csssel.py +++ /dev/null @@ -1,16 +0,0 @@ -from .unified import Selector - - -class CSSSelector(Selector): - - default_contenttype = 'html' - - def select(self, css): - return self.css(css) - - -HtmlCSSSelector = CSSSelector - - -class XmlCSSSelector(CSSSelector): - default_contenttype = 'xml' diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 12d0188ab..77cd9e11f 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -50,7 +50,7 @@ class Selector(object_ref): def __init__(self, response=None, text=None, namespaces=None, contenttype=None, _root=None, _expr=None): - self.contenttype = ct = self.default_contenttype or _ct(contenttype) + self.contenttype = ct = _ct(response, contenttype or self.default_contenttype) self._parser = _ctgroup[ct]['_parser'] self._csstranslator = _ctgroup[ct]['_csstranslator'] self._tostring_method = _ctgroup[ct]['_tostring_method'] @@ -146,8 +146,11 @@ class SelectorList(list): def __getslice__(self, i, j): return self.__class__(list.__getslice__(self, i, j)) - def select(self, xpath): - return self.__class__(flatten([x.select(xpath) for x in self])) + def xpath(self, xpath): + return self.__class__(flatten([x.xpath(xpath) for x in self])) + + def css(self, xpath): + return self.__class__(flatten([x.css(xpath) for x in self])) def re(self, regex): return flatten([x.re(regex) for x in self]) @@ -155,10 +158,15 @@ class SelectorList(list): def extract(self): return [x.extract() for x in self] - @deprecated(use_instead='SelectorList.extract') + @deprecated(use_instead='.extract()') def extract_unquoted(self): return [x.extract_unquoted() for x in self] - @deprecated(use_instead='SelectorList.select') + @deprecated(use_instead='.xpath()') def x(self, xpath): return self.select(xpath) + + @deprecated(use_instead='.xpath()') + def select(self, xpath): + return self.xpath(xpath) + diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index 399820240..dd2e292b6 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -2,9 +2,9 @@ Selector tests for cssselect backend """ from twisted.trial import unittest -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector +from scrapy.http import HtmlResponse from scrapy.selector.csstranslator import ScrapyHTMLTranslator +from scrapy.selector import Selector from cssselect.parser import SelectorSyntaxError from cssselect.xpath import ExpressionError @@ -116,20 +116,20 @@ class TranslatorMixinTest(unittest.TestCase): class HTMLCSSSelectorTest(unittest.TestCase): - hcs_cls = HtmlCSSSelector + hcs_cls = Selector def setUp(self): self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY) self.hcs = self.hcs_cls(self.htmlresponse) def x(self, *a, **kw): - return [v.strip() for v in self.hcs.select(*a, **kw).extract() if v.strip()] + return [v.strip() for v in self.hcs.css(*a, **kw).extract() if v.strip()] def test_selector_simple(self): - for x in self.hcs.select('input'): + for x in self.hcs.css('input'): self.assertTrue(isinstance(x, self.hcs.__class__), x) - self.assertEqual(self.hcs.select('input').extract(), - [x.extract() for x in self.hcs.select('input')]) + self.assertEqual(self.hcs.css('input').extract(), + [x.extract() for x in self.hcs.css('input')]) def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) @@ -147,7 +147,7 @@ class HTMLCSSSelectorTest(unittest.TestCase): self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.hcs.select('p').select('b::text').extract(), + self.assertEqual(self.hcs.css('p').css('b::text').extract(), [u'hi', u'guy']) - self.assertEqual(self.hcs.select('div').select('area:last-child').extract(), + self.assertEqual(self.hcs.css('div').css('area:last-child').extract(), [u'']) From a9eb0b74c8aebcbab72d5af20261c18bf8afc388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 11 Oct 2013 19:53:01 -0200 Subject: [PATCH 10/19] port scrapy shell to unified selector api --- scrapy/shell.py | 13 +++---------- scrapy/tests/test_command_shell.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index 642b3d6e6..8b6c209cb 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -11,7 +11,7 @@ from w3lib.url import any_to_uri from scrapy.item import BaseItem from scrapy.spider import BaseSpider -from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector, XmlCSSSelector, HtmlCSSSelector +from scrapy.selector import Selector from scrapy.utils.spider import create_spider_for_request from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser @@ -24,7 +24,7 @@ from scrapy.exceptions import IgnoreRequest class Shell(object): relevant_classes = (BaseSpider, Request, Response, BaseItem, - XPathSelector, Settings) + Selector, Settings) def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler @@ -95,14 +95,7 @@ class Shell(object): self.vars['spider'] = spider self.vars['request'] = request self.vars['response'] = response - self.vars['xxs'] = XmlXPathSelector(response) \ - if isinstance(response, XmlResponse) else None - self.vars['xcs'] = XmlCSSSelector(response) \ - if isinstance(response, XmlResponse) else None - self.vars['hxs'] = HtmlXPathSelector(response) \ - if isinstance(response, HtmlResponse) else None - self.vars['hcs'] = HtmlCSSSelector(response) \ - if isinstance(response, HtmlResponse) else None + self.vars['ss'] = Selector(response) if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser diff --git a/scrapy/tests/test_command_shell.py b/scrapy/tests/test_command_shell.py index bdfd8bb20..8dbaa632f 100644 --- a/scrapy/tests/test_command_shell.py +++ b/scrapy/tests/test_command_shell.py @@ -31,7 +31,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'hxs.select("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'ss.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), 'Works') From e4d6e2eb317026069467f99913531ee56a00908c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 11 Oct 2013 19:53:32 -0200 Subject: [PATCH 11/19] default xpath selector was html --- scrapy/selector/lxmlsel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index b37b5d74c..2052f0eaa 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -10,7 +10,7 @@ __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', class XPathSelector(Selector): __slots__ = () - default_contenttype = 'xml' + default_contenttype = 'html' class XmlXPathSelector(XPathSelector): From 4e94b38396535bef740672c7a9852ed427c6b34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 10:35:02 -0200 Subject: [PATCH 12/19] port tests to new Selector class --- scrapy/selector/lxmlsel.py | 15 +- scrapy/tests/test_selector.py | 293 +++++++++++------- ...lect.py => test_selector_csstranslator.py} | 20 +- scrapy/tests/test_selector_lxml.py | 62 ---- scrapy/tests/test_selector_lxmldocument.py | 26 ++ 5 files changed, 225 insertions(+), 191 deletions(-) rename scrapy/tests/{test_selector_cssselect.py => test_selector_csstranslator.py} (90%) delete mode 100644 scrapy/tests/test_selector_lxml.py create mode 100644 scrapy/tests/test_selector_lxmldocument.py diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 2052f0eaa..bc655b523 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -12,6 +12,19 @@ class XPathSelector(Selector): __slots__ = () default_contenttype = 'html' + def __init__(self, *a, **kw): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('%s is deprecated, instanciate scrapy.selector.Selector ' + 'instead' % type(self).__name__, + category=ScrapyDeprecationWarning, stacklevel=1) + super(XPathSelector, self).__init__(*a, **kw) + + def css(self, *a, **kw): + raise RuntimeError('.css() method not available for %s, ' + 'instanciate scrapy.selector.Selector ' + 'instead' % type(self).__name__) + class XmlXPathSelector(XPathSelector): __slots__ = () @@ -28,7 +41,7 @@ class XPathSelectorList(SelectorList): def __init__(self, *a, **kw): import warnings from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('XPathSelectorList is deprecated, use ' + warnings.warn('XPathSelectorList is deprecated, instanciate ' 'scrapy.selector.SelectorList instead', category=ScrapyDeprecationWarning, stacklevel=1) super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index fc732a694..4149b837c 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -1,81 +1,85 @@ -""" -Selectors tests, common for all backends -""" - import re +import warnings import weakref - from twisted.trial import unittest - +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector +from scrapy.selector import Selector +from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector -class XPathSelectorTestCase(unittest.TestCase): +class SelectorTestCase(unittest.TestCase): - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector + sscls = Selector - def test_selector_simple(self): + def test_simple_selection(self): """Simple selector tests""" body = "

    " response = TextResponse(url="http://example.com", body=body) - xpath = self.hxs_cls(response) + ss = self.sscls(response) - xl = xpath.select('//input') + xl = ss.xpath('//input') self.assertEqual(2, len(xl)) for x in xl: - assert isinstance(x, self.hxs_cls) + assert isinstance(x, self.sscls) - self.assertEqual(xpath.select('//input').extract(), - [x.extract() for x in xpath.select('//input')]) + self.assertEqual(ss.xpath('//input').extract(), + [x.extract() for x in ss.xpath('//input')]) - self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], + self.assertEqual([x.extract() for x in ss.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.extract() for x in ss.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), + self.assertEqual(ss.xpath("concat('xpath', 'rules')").extract(), [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.extract() for x in ss.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) - def test_selector_unicode_query(self): + def test_select_unicode_query(self): body = u"

    " response = TextResponse(url="http://example.com", body=body, encoding='utf8') - xpath = self.hxs_cls(response) - self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1']) + ss = self.sscls(response) + self.assertEqual(ss.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1']) - def test_selector_same_type(self): - """Test XPathSelector returning the same type in x() method""" + def test_list_elements_type(self): + """Test Selector returning the same type in selection methods""" text = '

    test

    ' - assert isinstance(self.xxs_cls(text=text).select("//p")[0], - self.xxs_cls) - assert isinstance(self.hxs_cls(text=text).select("//p")[0], - self.hxs_cls) + assert isinstance(self.sscls(text=text).xpath("//p")[0], self.sscls) + assert isinstance(self.sscls(text=text).css("p")[0], self.sscls) - def test_selector_boolean_result(self): + def test_boolean_result(self): body = "

    " response = TextResponse(url="http://example.com", body=body) - xs = self.hxs_cls(response) - self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1']) - self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0']) - - def test_selector_xml_html(self): - """Test that XML and HTML XPathSelector's behave differently""" + xs = self.sscls(response) + self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1']) + self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0']) + def test_differences_parsing_xml_vs_html(self): + """Test that XML and HTML Selector's behave differently""" # some text which is parsed differently by XML and HTML flavors text = '

    Hello

    ' - - self.assertEqual(self.xxs_cls(text=text).select("//div").extract(), - [u'

    Hello

    ']) - - self.assertEqual(self.hxs_cls(text=text).select("//div").extract(), + hs = self.sscls(text=text, contenttype='html') + self.assertEqual(hs.xpath("//div").extract(), [u'

    Hello

    ']) - def test_selector_nested(self): + xs = self.sscls(text=text, contenttype='xml') + self.assertEqual(xs.xpath("//div").extract(), + [u'

    Hello

    ']) + + def test_flavor_detection(self): + text = '

    Hello

    ' + ss = self.sscls(XmlResponse('http://example.com', body=text)) + self.assertEqual(ss.contenttype, 'xml') + self.assertEqual(ss.xpath("//div").extract(), + [u'

    Hello

    ']) + + ss = self.sscls(HtmlResponse('http://example.com', body=text)) + self.assertEqual(ss.contenttype, 'html') + self.assertEqual(ss.xpath("//div").extract(), + [u'

    Hello

    ']) + + def test_nested_selectors(self): """Nested selector tests""" body = """
    @@ -91,24 +95,30 @@ class XPathSelectorTestCase(unittest.TestCase): """ response = HtmlResponse(url="http://example.com", body=body) - x = self.hxs_cls(response) - - divtwo = x.select('//div[@class="two"]') - self.assertEqual(map(unicode.strip, divtwo.select("//li").extract()), + x = self.sscls(response) + divtwo = x.xpath('//div[@class="two"]') + self.assertEqual(divtwo.xpath("//li").extract(), ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(map(unicode.strip, divtwo.select("./ul/li").extract()), + self.assertEqual(divtwo.xpath("./ul/li").extract(), ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(map(unicode.strip, divtwo.select(".//li").extract()), + self.assertEqual(divtwo.xpath(".//li").extract(), ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select("./li").extract(), - []) + self.assertEqual(divtwo.xpath("./li").extract(), []) + + def test_mixed_nested_selectors(self): + body = ''' +
    notme
    +

    text

    foo
    + ''' + ss = self.sscls(text=body) + self.assertEqual(ss.xpath('//div[@id="1"]').css('span::text').extract(), [u'me']) + self.assertEqual(ss.css('#1').xpath('./span/text()').extract(), [u'me']) def test_dont_strip(self): - hxs = self.hxs_cls(text='
    fff: zzz
    ') - self.assertEqual(hxs.select("//text()").extract(), - [u'fff: ', u'zzz']) + hxs = self.sscls(text='
    fff: zzz
    ') + self.assertEqual(hxs.xpath("//text()").extract(), [u'fff: ', u'zzz']) - def test_selector_namespaces_simple(self): + def test_namespaces_simple(self): body = """ take this @@ -117,13 +127,13 @@ class XPathSelectorTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - x = self.xxs_cls(response) + x = self.sscls(response) x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.select("//somens:a/text()").extract(), + self.assertEqual(x.xpath("//somens:a/text()").extract(), [u'take this']) - def test_selector_namespaces_multiple(self): + def test_namespaces_multiple(self): body = """ """ response = XmlResponse(url="http://example.com", body=body) - x = self.xxs_cls(response) - + x = self.sscls(response) x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") x.register_namespace("p", "http://www.scrapy.org/product") x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.select("//xmlns:TestTag")), 1) - self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') + self.assertEqual(len(x.xpath("//xmlns:TestTag")), 1) + self.assertEqual(x.xpath("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.xpath("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.xpath("//p:SecondTestTag").xpath("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') - def test_selector_re(self): + def test_re(self): body = """
    Name: Mary
    • Name: John
      • @@ -155,44 +164,35 @@ class XPathSelectorTestCase(unittest.TestCase):
    • Age: 20
    Age: 20 -
    - - """ +
    """ response = HtmlResponse(url="http://example.com", body=body) - x = self.hxs_cls(response) + x = self.sscls(response) name_re = re.compile("Name: (\w+)") - self.assertEqual(x.select("//ul/li").re(name_re), + self.assertEqual(x.xpath("//ul/li").re(name_re), ["John", "Paul"]) - self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), + self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"), ["10", "20"]) - def test_selector_re_intl(self): + def test_re_intl(self): body = """
    Evento: cumplea\xc3\xb1os
    """ response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8') - x = self.hxs_cls(response) - self.assertEqual(x.select("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) + x = self.sscls(response) + self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) def test_selector_over_text(self): - hxs = self.hxs_cls(text='lala') - self.assertEqual(hxs.extract(), - u'lala') + hs = self.sscls(text='lala') + self.assertEqual(hs.extract(), u'lala') + xs = self.sscls(text='lala', contenttype='xml') + self.assertEqual(xs.extract(), u'lala') + self.assertEqual(xs.xpath('.').extract(), [u'lala']) - xxs = self.xxs_cls(text='lala') - self.assertEqual(xxs.extract(), - u'lala') - - xxs = self.xxs_cls(text='lala') - self.assertEqual(xxs.select('.').extract(), - [u'lala']) - - - def test_selector_invalid_xpath(self): + def test_invalid_xpath(self): response = XmlResponse(url="http://example.com", body="") - x = self.hxs_cls(response) + x = self.sscls(response) xpath = "//test[@foo='bar]" try: - x.select(xpath) + x.xpath(xpath) except ValueError, e: assert xpath in str(e), "Exception message does not contain invalid xpath" except Exception: @@ -215,64 +215,121 @@ class XPathSelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) - x = self.hxs_cls(response) - self.assertEquals(x.select("//span[@id='blank']/text()").extract(), + x = self.sscls(response) + self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_empty_bodies(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_null_bytes(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', \ body='pre\x00post', \ encoding='utf-8') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence r1 = TextResponse('http://www.example.com', \ body='

    an Jos\xe9 de

    ', \ encoding='utf-8') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_select_on_unevaluable_nodes(self): - r = self.hxs_cls(text=u'some text') + r = self.sscls(text=u'some text') # Text node - x1 = r.select('//text()') + x1 = r.xpath('//text()') self.assertEquals(x1.extract(), [u'some text']) - self.assertEquals(x1.select('.//b').extract(), []) + self.assertEquals(x1.xpath('.//b').extract(), []) # Tag attribute - x1 = r.select('//span/@class') + x1 = r.xpath('//span/@class') self.assertEquals(x1.extract(), [u'big']) - self.assertEquals(x1.select('.//text()').extract(), []) + self.assertEquals(x1.xpath('.//text()').extract(), []) def test_select_on_text_nodes(self): - r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') - x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") + r = self.sscls(text=u'
    Options:opt1
    Otheropt2
    ') + x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") self.assertEquals(x1.extract(), [u'opt1']) - x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") + x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x1.extract(), [u'Options:']) def test_nested_select_on_text_nodes(self): # FIXME: does not work with lxml backend [upstream] - r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') - x1 = r.select("//div/descendant::text()") - x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]") - + r = self.sscls(text=u'
    Options:opt1
    Otheropt2
    ') + x1 = r.xpath("//div/descendant::text()") + x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x2.extract(), [u'Options:']) - test_nested_select_on_text_nodes.skip = True + test_nested_select_on_text_nodes.skip = "Text nodes lost parent node reference in lxml" def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" - for cls in [self.xs_cls, self.hxs_cls, self.xxs_cls]: - x = cls() - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ + x = self.sscls() + weakref.ref(x) + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ + x.__class__.__name__ + + def test_remove_namespaces(self): + xml = """ + + + + +""" + xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(xxs.xpath("//link")), 0) + xxs.remove_namespaces() + self.assertEqual(len(xxs.xpath("//link")), 2) + + def test_remove_attributes_namespaces(self): + xml = """ + + + + +""" + xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(xxs.xpath("//link/@type")), 0) + xxs.remove_namespaces() + self.assertEqual(len(xxs.xpath("//link/@type")), 2) + + +class DeprecatedXpathSelectorTest(unittest.TestCase): + + text = '

    Hello

    ' + + def test_warnings(self): + for cls in XPathSelector, HtmlXPathSelector, XPathSelector: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + hs = cls(text=self.text) + assert len(w) == 1, w + assert issubclass(w[0].category, ScrapyDeprecationWarning) + assert 'deprecated' in str(w[-1].message) + hs.select("//div").extract() + assert issubclass(w[1].category, ScrapyDeprecationWarning) + assert 'deprecated' in str(w[-1].message) + + def test_xpathselector(self): + with warnings.catch_warnings(record=True): + hs = XPathSelector(text=self.text) + self.assertEqual(hs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, hs.css, 'div') + + def test_htmlxpathselector(self): + with warnings.catch_warnings(record=True): + hs = HtmlXPathSelector(text=self.text) + self.assertEqual(hs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, hs.css, 'div') + + def test_xmlxpathselector(self): + with warnings.catch_warnings(record=True): + xs = XmlXPathSelector(text=self.text) + self.assertEqual(xs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, xs.css, 'div') diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_csstranslator.py similarity index 90% rename from scrapy/tests/test_selector_cssselect.py rename to scrapy/tests/test_selector_csstranslator.py index dd2e292b6..45ef91c19 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_csstranslator.py @@ -114,22 +114,22 @@ class TranslatorMixinTest(unittest.TestCase): self.assertRaises(exc, self.c2x, css) -class HTMLCSSSelectorTest(unittest.TestCase): +class CSSSelectorTest(unittest.TestCase): - hcs_cls = Selector + sscls = Selector def setUp(self): self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY) - self.hcs = self.hcs_cls(self.htmlresponse) + self.ss = self.sscls(self.htmlresponse) def x(self, *a, **kw): - return [v.strip() for v in self.hcs.css(*a, **kw).extract() if v.strip()] + return [v.strip() for v in self.ss.css(*a, **kw).extract() if v.strip()] def test_selector_simple(self): - for x in self.hcs.css('input'): - self.assertTrue(isinstance(x, self.hcs.__class__), x) - self.assertEqual(self.hcs.css('input').extract(), - [x.extract() for x in self.hcs.css('input')]) + for x in self.ss.css('input'): + self.assertTrue(isinstance(x, self.ss.__class__), x) + self.assertEqual(self.ss.css('input').extract(), + [x.extract() for x in self.ss.css('input')]) def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) @@ -147,7 +147,7 @@ class HTMLCSSSelectorTest(unittest.TestCase): self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.hcs.css('p').css('b::text').extract(), + self.assertEqual(self.ss.css('p').css('b::text').extract(), [u'hi', u'guy']) - self.assertEqual(self.hcs.css('div').css('area:last-child').extract(), + self.assertEqual(self.ss.css('div').css('area:last-child').extract(), [u'']) diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py deleted file mode 100644 index 7ea674d62..000000000 --- a/scrapy/tests/test_selector_lxml.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Selectors tests, specific for lxml backend -""" - -import unittest -from scrapy.tests import test_selector -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector.lxmldocument import LxmlDocument -from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector - - -class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): - - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - - def test_remove_namespaces(self): - xml = """ - - - - -""" - xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.select("//link")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.select("//link")), 2) - - def test_remove_attributes_namespaces(self): - xml = """ - - - - -""" - xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.select("//link/@type")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.select("//link/@type")), 2) - - -class Libxml2DocumentTest(unittest.TestCase): - - def test_caching(self): - r1 = HtmlResponse('http://www.example.com', body='') - r2 = r1.copy() - - doc1 = LxmlDocument(r1) - doc2 = LxmlDocument(r1) - doc3 = LxmlDocument(r2) - - # make sure it's cached - assert doc1 is doc2 - assert doc1 is not doc3 - - def test_null_char(self): - # make sure bodies with null char ('\x00') don't raise a TypeError exception - self.body_content = 'test problematic \x00 body' - response = TextResponse('http://example.com/catalog/product/blabla-123', - headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) - LxmlDocument(response) diff --git a/scrapy/tests/test_selector_lxmldocument.py b/scrapy/tests/test_selector_lxmldocument.py new file mode 100644 index 000000000..e544613d5 --- /dev/null +++ b/scrapy/tests/test_selector_lxmldocument.py @@ -0,0 +1,26 @@ +import unittest +from scrapy.selector.lxmldocument import LxmlDocument +from scrapy.http import TextResponse, HtmlResponse + + +class Libxml2DocumentTest(unittest.TestCase): + + def test_caching(self): + r1 = HtmlResponse('http://www.example.com', body='') + r2 = r1.copy() + + doc1 = LxmlDocument(r1) + doc2 = LxmlDocument(r1) + doc3 = LxmlDocument(r2) + + # make sure it's cached + assert doc1 is doc2 + assert doc1 is not doc3 + + def test_null_char(self): + # make sure bodies with null char ('\x00') don't raise a TypeError exception + body = 'test problematic \x00 body' + response = TextResponse('http://example.com/catalog/product/blabla-123', + headers={'Content-Type': 'text/plain; charset=utf-8'}, + body=body) + LxmlDocument(response) From add3506928f78e001a200a0966925486ab67b84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 10:51:16 -0200 Subject: [PATCH 13/19] remove internal references to old selector classes and api --- scrapy/contrib/linkextractors/sgml.py | 6 +++--- scrapy/contrib/loader/__init__.py | 7 +++---- scrapy/contrib/spiders/feed.py | 12 ++++++------ scrapy/contrib_exp/iterators.py | 6 +++--- scrapy/tests/test_contrib_loader.py | 4 ++-- scrapy/tests/test_spider.py | 8 ++++---- scrapy/tests/test_utils_iterators.py | 26 +++++++++++++------------- scrapy/utils/iterators.py | 6 +++--- 8 files changed, 37 insertions(+), 38 deletions(-) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 7f2f4e4bc..f0e141a43 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -4,7 +4,7 @@ SGMLParser-based Link extractors import re from urlparse import urlparse, urljoin from w3lib.url import safe_url_string -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.link import Link from scrapy.linkextractor import IGNORED_EXTENSIONS from scrapy.utils.misc import arg_to_iter @@ -116,11 +116,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): base_url = None if self.restrict_xpaths: - hxs = HtmlXPathSelector(response) + ss = Selector(response) base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths - for f in hxs.select(x).extract() + for f in ss.xpath(x).extract() ).encode(response.encoding) else: body = response.body diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index fa8387eaa..ecd20fc3d 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -8,7 +8,7 @@ from collections import defaultdict import re from scrapy.item import Item -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten from .common import wrap_loader_context @@ -116,7 +116,7 @@ class ItemLoader(object): class XPathItemLoader(ItemLoader): - default_selector_class = HtmlXPathSelector + default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, **context): if selector is None and response is None: @@ -142,5 +142,4 @@ class XPathItemLoader(ItemLoader): def _get_values(self, xpaths, **kw): xpaths = arg_to_iter(xpaths) - return flatten([self.selector.select(xpath).extract() for xpath in xpaths]) - + return flatten([self.selector.xpath(xpath).extract() for xpath in xpaths]) diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 89c6277ab..6d8e0a358 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -9,7 +9,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.utils.iterators import xmliter, csviter from scrapy.utils.spider import iterate_spider_output -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector +from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported @@ -52,7 +52,7 @@ class XMLFeedSpider(BaseSpider): def parse_nodes(self, response, nodes): """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an XPathSelector for each node. + (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return either a BaseItem, a Request, or a list containing any of them. @@ -71,13 +71,13 @@ class XMLFeedSpider(BaseSpider): if self.iterator == 'iternodes': nodes = self._iternodes(response) elif self.iterator == 'xml': - selector = XmlXPathSelector(response) + selector = Selector(response, contenttype='xml') self._register_namespaces(selector) - nodes = selector.select('//%s' % self.itertag) + nodes = selector.xpath('//%s' % self.itertag) elif self.iterator == 'html': - selector = HtmlXPathSelector(response) + selector = Selector(response, contenttype='html') self._register_namespaces(selector) - nodes = selector.select('//%s' % self.itertag) + nodes = selector.xpath('//%s' % self.itertag) else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py index 0f3a8c694..ecc6265d0 100644 --- a/scrapy/contrib_exp/iterators.py +++ b/scrapy/contrib_exp/iterators.py @@ -1,5 +1,5 @@ from scrapy.http import Response -from scrapy.selector import XmlXPathSelector +from scrapy.selector import Selector def xmliter_lxml(obj, nodename, namespace=None): @@ -11,10 +11,10 @@ def xmliter_lxml(obj, nodename, namespace=None): for _, node in iterable: nodetext = etree.tostring(node) node.clear() - xs = XmlXPathSelector(text=nodetext) + xs = Selector(text=nodetext, contenttype='xml') if namespace: xs.register_namespace('x', namespace) - yield xs.select(selxpath)[0] + yield xs.xpath(selxpath)[0] class _StreamReader(object): diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 80f3151da..976df5f12 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -4,7 +4,7 @@ from scrapy.contrib.loader import ItemLoader, XPathItemLoader from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \ Compose, MapCompose from scrapy.item import Item, Field -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.http import HtmlResponse @@ -379,7 +379,7 @@ class XPathItemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, XPathItemLoader) def test_constructor_with_selector(self): - sel = HtmlXPathSelector(text=u"
    marta
    ") + sel = Selector(text=u"
    marta
    ") l = TestXPathItemLoader(selector=sel) self.assert_(l.selector is sel) l.add_xpath('name', '//div/text()') diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 9ffa2d7cc..97076089b 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -70,10 +70,10 @@ class XMLFeedSpiderTest(BaseSpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.select('a:loc/text()').extract(), - 'updated': selector.select('b:updated/text()').extract(), - 'other': selector.select('other/@value').extract(), - 'custom': selector.select('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').extract(), + 'updated': selector.xpath('b:updated/text()').extract(), + 'other': selector.xpath('other/@value').extract(), + 'custom': selector.xpath('other/@b:custom').extract(), } for iterator in ('iternodes', 'xml'): diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 21963031b..4ddd05693 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -28,7 +28,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract())) + attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) @@ -36,7 +36,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_text(self): body = u"""onetwo""" - self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')], + self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -63,15 +63,15 @@ class XmliterTestCase(unittest.TestCase): node = my_iter.next() node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.select('title/text()').extract(), ['Item 1']) - self.assertEqual(node.select('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.select('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.select('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.select('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.select('g:price/text()').extract(), ['400']) - self.assertEqual(node.select('image_link/text()').extract(), []) - self.assertEqual(node.select('id/text()').extract(), []) - self.assertEqual(node.select('price/text()').extract(), []) + self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) + self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1']) + self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1']) + self.assertEqual(node.xpath('g:price/text()').extract(), ['400']) + self.assertEqual(node.xpath('image_link/text()').extract(), []) + self.assertEqual(node.xpath('id/text()').extract(), []) + self.assertEqual(node.xpath('price/text()').extract(), []) def test_xmliter_exception(self): body = u"""onetwo""" @@ -123,9 +123,9 @@ class LxmlXmliterTestCase(XmliterTestCase): namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') node = namespace_iter.next() - self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) node = namespace_iter.next() - self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) class UtilsCsvTestCase(unittest.TestCase): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 327b9e024..67f9946e6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -2,14 +2,14 @@ import re, csv from cStringIO import StringIO from scrapy.http import TextResponse -from scrapy.selector import XmlXPathSelector +from scrapy.selector import Selector from scrapy import log from scrapy.utils.python import re_rsearch, str_to_unicode from scrapy.utils.response import body_or_str def xmliter(obj, nodename): - """Return a iterator of XPathSelector's over all nodes of a XML document, + """Return a iterator of Selector's over all nodes of a XML document, given tha name of the node to iterate. Useful for parsing XML feeds. obj can be: @@ -29,7 +29,7 @@ def xmliter(obj, nodename): r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end - yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] + yield Selector(text=nodetext, contenttype='xml').xpath('//' + nodename)[0] def csviter(obj, delimiter=None, headers=None, encoding=None): From 4645f9e03cf1d0ebe9ad4dd71d1fd6b725c583cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 16:31:20 -0200 Subject: [PATCH 14/19] Updates docs to reflect unified selectors api --- docs/intro/overview.rst | 9 +- docs/intro/tutorial.rst | 98 ++++----- docs/topics/firebug.rst | 12 +- docs/topics/leaks.rst | 6 +- docs/topics/loaders.rst | 10 +- docs/topics/selectors.rst | 422 +++++++++++++------------------------- docs/topics/shell.rst | 29 ++- docs/topics/spiders.rst | 47 ++--- sep/sep-020.rst | 8 +- 9 files changed, 254 insertions(+), 387 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index eed4bdac4..2138e76f8 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -143,13 +143,12 @@ Finally, here's the spider code:: rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] def parse_torrent(self, response): - x = HtmlXPathSelector(response) - + ss = Selector(response) torrent = TorrentItem() torrent['url'] = response.url - torrent['name'] = x.select("//h1/text()").extract() - torrent['description'] = x.select("//div[@id='description']").extract() - torrent['size'] = x.select("//div[@id='info-left']/p[2]/text()[2]").extract() + torrent['name'] = ss.xpath("//h1/text()").extract() + torrent['description'] = ss.xpath("//div[@id='description']").extract() + torrent['size'] = ss.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() return torrent For brevity's sake, we intentionally left out the import statements. The diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 495f8b9f8..5068cd8c1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -183,11 +183,12 @@ Introduction to Selectors ^^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to extract data from web pages. Scrapy uses a mechanism -based on `XPath`_ expressions called :ref:`XPath selectors `. -For more information about selectors and other extraction mechanisms see the -:ref:`XPath selectors documentation `. +based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors +`. For more information about selectors and other extraction +mechanisms see the :ref:`Selectors documentation `. .. _XPath: http://www.w3.org/TR/xpath +.. _CSS: http://www.w3.org/TR/selectors Here are some examples of XPath expressions and their meanings: @@ -206,27 +207,28 @@ These are just a couple of simple examples of what you can do with XPath, but XPath expressions are indeed much more powerful. To learn more about XPath we recommend `this XPath tutorial `_. -For working with XPaths, Scrapy provides a :class:`~scrapy.selector.XPathSelector` -class, which comes in two flavours, :class:`~scrapy.selector.HtmlXPathSelector` -(for HTML data) and :class:`~scrapy.selector.XmlXPathSelector` (for XML data). In -order to use them you must instantiate the desired class with a -:class:`~scrapy.http.Response` object. +For working with XPaths, Scrapy provides a :class:`~scrapy.selector.Selector` +class, it must be instantiated with a :class:`~scrapy.http.HtmlResponse` or +:class:`~scrapy.http.XmlResponse` object as first argument. You can see selectors as objects that represent nodes in the document structure. So, the first instantiated selectors are associated to the root node, or the entire document. -Selectors have three methods (click on the method to see the complete API +Selectors have four basic methods (click on the method to see the complete API documentation). -* :meth:`~scrapy.selector.XPathSelector.select`: returns a list of selectors, each of +* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of them representing the nodes selected by the xpath expression given as - argument. + argument. -* :meth:`~scrapy.selector.XPathSelector.extract`: returns a unicode string with - the data selected by the XPath selector. +* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of + them representing the nodes selected by the CSS expression given as argument. -* :meth:`~scrapy.selector.XPathSelector.re`: returns a list of unicode strings +* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the + selected data. + +* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings extracted by applying the regular expression given as argument. @@ -253,12 +255,11 @@ This is what the shell looks like:: [s] Available Scrapy objects: [s] 2010-08-19 21:45:59-0300 [default] INFO: Spider closed (finished) - [s] hxs + [s] ss [s] item Item() [s] request [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Print this help [s] fetch(req_or_url) Fetch a new request or URL and update shell objects @@ -270,23 +271,24 @@ After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body of the response, or you can type ``response.headers`` to see its headers. -The shell also instantiates two selectors, one for HTML (in the ``hxs`` -variable) and one for XML (in the ``xxs`` variable) with this response. So let's -try them:: +The shell also pre-instantiate a selector named ``ss``, it automatically choice +the best parsing rules (XML vs HTML) based on response's type. - In [1]: hxs.select('//title') - Out[1]: [] +So let's try it:: - In [2]: hxs.select('//title').extract() + In [1]: ss.xpath('//title') + Out[1]: [] + + In [2]: ss.xpath('//title').extract() Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: hxs.select('//title/text()') - Out[3]: [] + In [3]: ss.xpath('//title/text()') + Out[3]: [] - In [4]: hxs.select('//title/text()').extract() + In [4]: ss.xpath('//title/text()').extract() Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: hxs.select('//title/text()').re('(\w+):') + In [5]: ss.xpath('//title/text()').re('(\w+):') Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data @@ -306,29 +308,29 @@ is inside a ``
      `` element, in fact the *second* ``
        `` element. So we can select each ``
      • `` element belonging to the sites list with this code:: - hxs.select('//ul/li') + ss.xpath('//ul/li') And from them, the sites descriptions:: - hxs.select('//ul/li/text()').extract() + ss.xpath('//ul/li/text()').extract() The sites titles:: - hxs.select('//ul/li/a/text()').extract() + ss.xpath('//ul/li/a/text()').extract() And the sites links:: - hxs.select('//ul/li/a/@href').extract() + ss.xpath('//ul/li/a/@href').extract() -As we said before, each ``select()`` call returns a list of selectors, so we can -concatenate further ``select()`` calls to dig deeper into a node. We are going to use +As we said before, each ``.xpath()`` call returns a list of selectors, so we can +concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use that property here, so:: - sites = hxs.select('//ul/li') + sites = ss.xpath('//ul/li') for site in sites: - title = site.select('a/text()').extract() - link = site.select('a/@href').extract() - desc = site.select('text()').extract() + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() print title, link, desc .. note:: @@ -341,7 +343,7 @@ that property here, so:: Let's add this code to our spider:: from scrapy.spider import BaseSpider - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector class DmozSpider(BaseSpider): name = "dmoz" @@ -352,12 +354,12 @@ Let's add this code to our spider:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - sites = hxs.select('//ul/li') + ss = Selector(response) + sites = ss.xpath('//ul/li') for site in sites: - title = site.select('a/text()').extract() - link = site.select('a/@href').extract() - desc = site.select('text()').extract() + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() print title, link, desc Now try crawling the dmoz.org domain again and you'll see sites being printed @@ -382,7 +384,7 @@ Spiders are expected to return their scraped data inside scraped so far, the final code for our Spider would be like this:: from scrapy.spider import BaseSpider - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from tutorial.items import DmozItem @@ -395,14 +397,14 @@ scraped so far, the final code for our Spider would be like this:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - sites = hxs.select('//ul/li') + ss = Selector(response) + sites = ss.xpath('//ul/li') items = [] for site in sites: item = DmozItem() - item['title'] = site.select('a/text()').extract() - item['link'] = site.select('a/@href').extract() - item['desc'] = site.select('text()').extract() + item['title'] = site.xpath('a/text()').extract() + item['link'] = site.xpath('a/@href').extract() + item['desc'] = site.xpath('text()').extract() items.append(item) return items diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 395f3bfe1..839701a32 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -107,7 +107,7 @@ Now we're going to write the code to extract data from those pages. With the help of Firebug, we'll take a look at some page containing links to websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can -extract those links using :ref:`XPath selectors `. We'll also +extract those links using :ref:`Selectors `. We'll also use the :ref:`Scrapy shell ` to test those XPath's and make sure they work as we expect. @@ -146,16 +146,16 @@ that have that grey colour of the links, Finally, we can write our ``parse_category()`` method:: def parse_category(self, response): - hxs = HtmlXPathSelector(response) + ss = Selector(response) # The path to website links in directory page - links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + links = ss.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') for link in links: item = DirectoryItem() - item['name'] = link.select('a/text()').extract() - item['url'] = link.select('a/@href').extract() - item['description'] = link.select('font[2]/text()').extract() + item['name'] = link.xpath('a/text()').extract() + item['url'] = link.xpath('a/@href').extract() + item['description'] = link.xpath('font[2]/text()').extract() yield item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 95a41f360..de9bf645e 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -67,7 +67,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:: ExampleSpider 1 oldest: 15s ago HtmlResponse 10 oldest: 1s ago - XPathSelector 2 oldest: 0s ago + Selector 2 oldest: 0s ago FormRequest 878 oldest: 7s ago As you can see, that report also shows the "age" of the oldest object in each @@ -87,7 +87,7 @@ subclasses): * ``scrapy.http.Request`` * ``scrapy.http.Response`` * ``scrapy.item.Item`` -* ``scrapy.selector.XPathSelector`` +* ``scrapy.selector.Selector`` * ``scrapy.spider.BaseSpider`` * ``scrapy.selector.document.Libxml2Document`` @@ -117,7 +117,7 @@ references:: SomenastySpider 1 oldest: 15s ago HtmlResponse 3890 oldest: 265s ago - XPathSelector 2 oldest: 0s ago + Selector 2 oldest: 0s ago Request 3878 oldest: 250s ago The fact that there are so many live responses (and that they're so old) is diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index bab3d6113..47c06896b 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -31,7 +31,7 @@ using the Item class specified in the :attr:`ItemLoader.default_item_class` attribute. Then, you start collecting values into the Item Loader, typically using -:ref:`XPath Selectors `. You can add more than one value to +:ref:`Selectors `. You can add more than one value to the same item field; the Item Loader will know how to "join" those values later using a proper processing function. @@ -352,14 +352,14 @@ ItemLoader objects The :class:`XPathItemLoader` class extends the :class:`ItemLoader` class providing more convenient mechanisms for extracting data from web pages - using :ref:`XPath selectors `. + using :ref:`selectors `. :class:`XPathItemLoader` objects accept two more additional parameters in their constructors: :param selector: The selector to extract data from, when using the :meth:`add_xpath` or :meth:`replace_xpath` method. - :type selector: :class:`~scrapy.selector.XPathSelector` object + :type selector: :class:`~scrapy.selector.Selector` object :param response: The response used to construct the selector using the :attr:`default_selector_class`, unless the selector argument is given, @@ -418,7 +418,7 @@ ItemLoader objects .. attribute:: selector - The :class:`~scrapy.selector.XPathSelector` object to extract data from. + The :class:`~scrapy.selector.Selector` object to extract data from. It's either the selector given in the constructor or one created from the response given in the constructor using the :attr:`default_selector_class`. This attribute is meant to be @@ -592,7 +592,7 @@ Here is a list of all built-in processors: work with single values (instead of iterables). For this reason the :class:`MapCompose` processor is typically used as input processor, since data is often extracted using the - :meth:`~scrapy.selector.XPathSelector.extract` method of :ref:`selectors + :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors `, which returns a list of unicode strings. The example below should clarify how it works:: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 554f16eb6..b9b185d4c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -33,9 +33,8 @@ small and simple, unlike the `lxml`_ API which is much bigger because the `lxml`_ library can be used for many other tasks, besides selecting markup documents. -For a complete reference of the selectors API see :ref:`XPath selector -reference ` and :ref:`CSS selector reference -`. +For a complete reference of the selectors API see +:ref:`Selector reference ` .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://codespeak.net/lxml/ @@ -44,36 +43,33 @@ reference ` and :ref:`CSS selector reference .. _XPath: http://www.w3.org/TR/xpath .. _CSS: http://www.w3.org/TR/selectors + Using selectors =============== Constructing selectors ---------------------- -There are four types of selectors bundled with Scrapy. Those are: - - * :class:`~scrapy.selector.HtmlXPathSelector` - for working with HTML - documents using XPath. - - * :class:`~scrapy.selector.XmlXPathSelector` - for working with XML documents - using XPath. - - * :class:`~scrapy.selector.HtmlCSSSelector` - for working with HTML documents - using CSS selectors. - - * :class:`~scrapy.selector.XmlCSSSelector` - for working with XML documents - using CSS selectors. - .. highlight:: python -All of them share the same selector API, and are constructed with a Response -object as their first parameter. This is the Response they're going to be -"selecting". +Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +constructed by passing a `Response` object as first argument, the response's +body is what they're going to be "selecting":: -Example:: + from scrapy.spider import BaseSpider + from scrapy.selector import Selector + + class MySpider(BaseSpider): + # ... + def parse(self, response): + ss = Selector(response) + # Using XPath query + print ss.xpath('//p') + # Using CSS query + print ss.css('p') + # Nesting queries + print ss.xpath('//div[@foo="bar"]').css('span#bold') - hcs = HtmlCSSSelector(response) # an HTML CSS selector - xxs = XmlXPathSelector(response) # an XML XPath selector Using selectors --------------- @@ -97,13 +93,10 @@ First, let's open the shell:: scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html -Then, after the shell loads, you'll have some selectors already instantiated -and ready to use. +Then, after the shell loads, you'll have a selector already instantiated and +ready to use in ``ss`` shell variable. -Since we're dealing with HTML, we can use either the -:class:`~scrapy.selector.HtmlXPathSelector` object which is found, by default, -in the ``hxs`` shell variable, or the equivalent -:class:`~scrapy.selector.HtmlCSSSelector` found in the ``hcs`` shell variable. +Since we're dealing with HTML, the selector will automatically use an HTML parser. .. highlight:: python @@ -111,57 +104,55 @@ So, by looking at the :ref:`HTML code ` of that page, let's construct an XPath (using an HTML selector) for selecting the text inside the title tag:: - >>> hxs.select('//title/text()') - [] + >>> ss.xpath('//title/text()') + [] -As you can see, the ``select()`` method returns an -:class:`~scrapy.selector.SelectorList`, which is a list of new selectors. This -API can be used quickly for extracting nested data. +As you can see, the ``.xpath()`` method returns an +:class:`~scrapy.selector.SelectorList` instance, which is a list of new +selectors. This API can be used quickly for extracting nested data. -To actually extract the textual data, you must call the selector ``extract()`` +To actually extract the textual data, you must call the selector ``.extract()`` method, as follows:: - >>> hxs.select('//title/text()').extract() + >>> ss.xpath('//title/text()').extract() [u'Example website'] -Now notice that CSS selectors can select text or attribute nodes using CSS3 +Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: - >>> hcs.select('title::text') - [] - >>> hcs.select('title::text').extract() + >>> ss.css('title::text').extract() [u'Example website'] Now we're going to get the base URL and some image links:: - >>> hxs.select('//base/@href').extract() + >>> ss.xpath('//base/@href').extract() [u'http://example.com/'] - >>> hcs.select('base::attr(href)').extract() + >>> ss.css('base::attr(href)').extract() [u'http://example.com/'] - >>> hxs.select('//a[contains(@href, "image")]/@href').extract() + >>> ss.xpath('//a[contains(@href, "image")]/@href').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> hcs.select('a[href*=image]::attr(href)').extract() + >>> ss.css('a[href*=image]::attr(href)').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> hxs.select('//a[contains(@href, "image")]/img/@src').extract() + >>> ss.xpath('//a[contains(@href, "image")]/img/@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> hcs.select('a[href*=image] img::attr(src)').extract() + >>> ss.css('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -173,11 +164,11 @@ Now we're going to get the base URL and some image links:: Nesting selectors ----------------- -The ``select()`` selector method returns a list of selectors of the same type -(XPath or CSS), so you can call the ``select()`` for those selectors too. -Here's an example:: +The selection methods (``.xpath()`` or ``.css()``) returns a list of selectors +of the same type, so you can call the selection methods for those selectors +too. Here's an example:: - >>> links = hxs.select('//a[contains(@href, "image")]') + >>> links = ss.xpath('//a[contains(@href, "image")]') >>> links.extract() [u'Name: My image 1
        ', u'Name: My image 2
        ', @@ -186,7 +177,7 @@ Here's an example:: u'Name: My image 5
        '] >>> for index, link in enumerate(links): - args = (index, link.select('@href').extract(), link.select('img/@src').extract()) + args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract()) print 'Link number %d points to url %s and image %s' % args Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] @@ -198,16 +189,15 @@ Here's an example:: Using selectors with regular expressions ---------------------------------------- -Selectors (both CSS and XPath) also have a ``re()`` method for extracting data -using regular expressions. However, unlike using the ``select()`` method, the -``re()`` method does not return a list of -:class:`Selector` objects, so you can't construct nested -``.re()`` calls. +:class:`~scrapy.selector.Selector` also have a ``.re()`` method for extracting +data using regular expressions. However, unlike using ``.xpath()`` or +``.css()`` methods, ``.re()`` method returns a list of unicode strings. So you +can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code ` above:: - >>> hxs.select('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + >>> ss.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') [u'My image 1', u'My image 2', u'My image 3', @@ -219,30 +209,30 @@ Here's an example used to extract images names from the :ref:`HTML code Working with relative XPaths ---------------------------- -Keep in mind that if you are nesting XPathSelectors and use an XPath that -starts with ``/``, that XPath will be absolute to the document and not relative -to the ``XPathSelector`` you're calling it from. +Keep in mind that if you are nesting selectors and use an XPath that starts +with ``/``, that XPath will be absolute to the document and not relative to the +``Selector`` you're calling it from. For example, suppose you want to extract all ``

        `` elements inside ``

        `` elements. First, you would get all ``
        `` elements:: - >>> divs = hxs.select('//div') + >>> divs = ss.xpath('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``

        `` elements from the document, not only those inside ``

        `` elements:: - >>> for p in divs.select('//p') # this is wrong - gets all

        from the whole document + >>> for p in divs.xpath('//p') # this is wrong - gets all

        from the whole document >>> print p.extract() This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: - >>> for p in divs.select('.//p') # extracts all

        inside + >>> for p in divs.xpath('.//p') # extracts all

        inside >>> print p.extract() Another common case would be to extract all direct ``

        `` children:: - >>> for p in divs.select('p') + >>> for p in divs.xpath('p') >>> print p.extract() For more details about relative XPaths see the `Location Paths`_ section in the @@ -257,42 +247,67 @@ Built-in Selectors reference ============================ .. module:: scrapy.selector - :synopsis: Selectors classes + :synopsis: Selector class -There are four types of selectors bundled with Scrapy: -:class:`HtmlXPathSelector` and :class:`XmlXPathSelector`, -:class:`HtmlCSSSelector` and :class:`XmlCSSSelector`. +.. class:: Selector(response, contenttype=None) -All of them implement the same :class:`XPathSelector` interface. The only -differences are the selector syntax and whether it is used to process HTML data -or XML data. + An instance of :class:`Selector` is a wrapper over ``response`` to select + certain parts of its content. -Selector interface ------------------- + ``response`` is a :class:`~scrapy.http.HtmlResponse` or + :class:`~scrapy.http.XmlResponse` object that will be used for selecting and + extracting data. -.. class:: Selector(response) + ``contenttype`` tells what parser and selection flavor is used to parse the + response body. It defaults to ``"html"`` for :class:`~scrapy.http.HtmlResponse` + and ``"xml"`` for :class:`~scrapy.http.XmlResponse`. - An instance implementing :class:`Selector` interface is a wrapper over - ``response`` to select certain parts of its content. + .. method:: xpath(query) - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. + Find nodes matching the xpath ``query`` and return the result as a + :class:`SelectorList` instance with all elements flattened. List + elements implement :class:`Selector` interface too. - .. method:: select(query) + ``query`` is a string containing the XPATH query to apply. - Find nodes matching the selection query and return the result as a - :class:`SelectorList` instance with all elements flattened. List - elements must implement :class:`Selector` interface too. + .. method:: css(query) - .. method:: extract() + Apply the given CSS selector and return a :class:`SelectorList` instance. - Serialize and return the matched nodes as a list of unicode strings + ``query`` is a string containing the CSS selector to apply. - .. method:: __nonzero__() + In the background, CSS queries are translated into XPath queries using + `cssselect`_ library and run ``.xpath()`` method. - Returns ``True`` if there is any real content selected or ``False`` - otherwise. In other words, the boolean value of a :class:`Selector` is - given by the contents it selects. + .. method:: extract() + + Serialize and return the matched nodes as a list of unicode strings. + Percent encoded content is unquoted. + + .. method:: re(regex) + + Apply the given regex and return a list of unicode strings with the + matches. + + ``regex`` can be either a compiled regular expression or a string which + will be compiled to a regular expression using ``re.compile(regex)`` + + .. method:: register_namespace(prefix, uri) + + Register the given namespace to be used in this :class:`Selector`. + Without registering namespaces you can't select or extract data from + non-standard namespaces. See examples below. + + .. method:: remove_namespaces() + + Remove all namespaces, allowing to traverse the document using + namespace-less xpaths. See example below. + + .. method:: __nonzero__() + + Returns ``True`` if there is any real content selected or ``False`` + otherwise. In other words, the boolean value of a :class:`Selector` is + given by the contents it selects. SelectorList objects @@ -303,16 +318,28 @@ SelectorList objects The :class:`SelectorList` class is subclass of the builtin ``list`` class, which provides a few additional methods. - .. method:: select(query) + .. method:: xpath(query) - Call the ``select()`` method for each element in this list and return + Call the ``.xpath()`` method for each element in this list and return their results flattened as another :class:`SelectorList`. - ``query`` is the same argument as the one in :meth:`Selector.select` + ``query`` is the same argument as the one in :meth:`Selector.xpath` + + .. method:: css(query) + + Call the ``.css()`` method for each element in this list and return + their results flattened as another :class:`SelectorList`. + + ``query`` is the same argument as the one in :meth:`Selector.css` .. method:: extract() - Call the ``extract()`` method for each element is this list and return + Call the ``.extract()`` method for each element is this list and return + their results flattened, as a list of unicode strings. + + .. method:: re() + + Call the ``.re()`` method for each element is this list and return their results flattened, as a list of unicode strings. .. method:: __nonzero__() @@ -320,126 +347,50 @@ SelectorList objects returns True if the list is not empty, False otherwise. -.. _topics-xpath-selectors-ref: +Selector examples on HTML response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -XPathSelector objects ---------------------- +Here's a couple of :class:`Selector` examples to illustrate several concepts. +In all cases, we assume there is already an :class:`Selector` instantiated with +a :class:`~scrapy.http.HtmlResponse` object like this:: -.. class:: XPathSelector(response) - - :class:`Selector` interface implementation that uses `XPath`_ query language - to select content on ``response`` - - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. - - In the background, XPath selectors are powered by `lxml`_ library - - .. method:: select(xpath) - - Apply the given XPath relative to this XPathSelector and return a list - of :class:`XPathSelector` objects (ie. a :class:`SelectorList`) - with the result. - - ``xpath`` is a string containing the XPath to apply - - .. method:: re(regex) - - Apply the given regex and return a list of unicode strings with the - matches. - - ``regex`` can be either a compiled regular expression or a string which - will be compiled to a regular expression using ``re.compile(regex)`` - - .. method:: extract() - - Return a unicode string with the content of this :class:`XPathSelector` - object. - - .. method:: register_namespace(prefix, uri) - - Register the given namespace to be used in this :class:`XPathSelector`. - Without registering namespaces you can't select or extract data from - non-standard namespaces. See examples below. - - .. method:: remove_namespaces() - - Remove all namespaces, allowing to traverse the document using - namespace-less xpaths. See example below. - - -HtmlXPathSelector objects -------------------------- - -.. class:: HtmlXPathSelector(response) - - A subclass of :class:`XPathSelector` for working with HTML content. It uses - the `lxml`_ HTML parser. See the :class:`XPathSelector` API for more - info. - -HtmlXPathSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`HtmlXPathSelector` examples to illustrate several -concepts. In all cases, we assume there is already an -:class:`HtmlXPathSelector` instantiated with a :class:`~scrapy.http.Response` -object like this:: - - x = HtmlXPathSelector(html_response) + x = Selector(html_response) 1. Select all ``

        `` elements from a HTML response body, returning a list of - :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` object):: + :class:`Selector` objects (ie. a :class:`SelectorList` object):: - x.select("//h1") + x.xpath("//h1") 2. Extract the text of all ``

        `` elements from a HTML response body, returning a list of unicode strings:: - x.select("//h1").extract() # this includes the h1 tag - x.select("//h1/text()").extract() # this excludes the h1 tag + x.xpath("//h1").extract() # this includes the h1 tag + x.xpath("//h1/text()").extract() # this excludes the h1 tag 3. Iterate over all ``

        `` tags and print their class attribute:: - for node in x.select("//p"): - ... print node.select("@class").extract() + for node in x.xpath("//p"): + ... print node.xpath("@class").extract() -4. Extract textual data from all ``

        `` tags without entities, as a list of - unicode strings:: +Selector examples on XML response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - x.select("//p/text()").extract_unquoted() +Here's a couple of examples to illustrate several concepts. In both cases we +assume there is already an :class:`Selector` instantiated with a +:class:`~scrapy.http.XmlResponse` object like this:: - # the following line is wrong. extract_unquoted() should only be used - # with textual XPathSelectors - x.select("//p").extract_unquoted() # it may work but output is unpredictable - -XmlXPathSelector objects ------------------------- - -.. class:: XmlXPathSelector(response) - - A subclass of :class:`XPathSelector` for working with XML content. It uses - the `lxml`_ XML parser. See the :class:`XPathSelector` API for more info. - -XmlXPathSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`XmlXPathSelector` examples to illustrate several -concepts. In both cases we assume there is already an :class:`XmlXPathSelector` -instantiated with a :class:`~scrapy.http.Response` object like this:: - - x = XmlXPathSelector(xml_response) + x = Selector(xml_response) 1. Select all ```` elements from a XML response body, returning a list - of :class:`XPathSelector` objects (ie. a :class:`XPathSelectorList` - object):: + of :class:`Selector` objects (ie. a :class:`SelectorList` object):: - x.select("//product") + x.xpath("//product") 2. Extract all prices from a `Google Base XML feed`_ which requires registering a namespace:: x.register_namespace("g", "http://base.google.com/ns/1.0") - x.select("//g:price").extract() + x.xpath("//g:price").extract() .. _removing-namespaces: @@ -449,7 +400,7 @@ Removing namespaces When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the -:meth:`XPathSelector.remove_namespaces` method for that. +:meth:`Selector.remove_namespaces` method for that. Let's show an example that illustrates this with Github blog atom feed. @@ -460,16 +411,16 @@ First, we open the shell with the url we want to scrape:: Once in the shell we can try selecting all ```` objects and see that it doesn't work (because the Atom XML namespace is obfuscating those nodes):: - >>> xxs.select("//link") + >>> xxs.xpath("//link") [] -But once we call the :meth:`XPathSelector.remove_namespaces` method, all +But once we call the :meth:`Selector.remove_namespaces` method, all nodes can be accessed directly by their names:: >>> xxs.remove_namespaces() - >>> xxs.select("//link") - [, + >>> xxs.xpath("//link") + [, ... If you wonder why the namespace removal procedure is not always called, instead @@ -485,84 +436,3 @@ of relevance, are: though. .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 - -.. _topics-css-selectors-ref: - -CSSSelector objects -------------------- - -.. class:: CSSSelector(response) - - :class:`Selector` interface implementation that uses `CSS`_ query language - to select content on ``response`` - - ``response`` is a :class:`~scrapy.http.Response` object that will be used - for selecting and extracting data. - - In the background, CSS selectors are translated into XPath selectors using - `cssselect`_ library and run using :class:`XPathSelector` - - .. method:: select(css) - - Apply the given CSS selector relative to this CSSSelector and return a - :class:`SelectorList` instance. - - ``css`` is a string containing the CSS selector to apply. - -HtmlCSSSelector objects ------------------------ - -.. class:: HtmlCSSSelector(response) - - A specialized class for working with HTML content using `CSS`_ selectors. - -HtmlCSSSelector examples -~~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`HtmlCSSSelector` examples to illustrate several -concepts. In all cases, we assume there is already an :class:`HtmlCSSSelector` -instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: - - x = HtmlCSSSelector(html_response) - -1. Select all ``

        `` elements from a HTML response body, returning a list of - :class:`HtmlCSSSelector` objects:: - - x.select("h1") - -2. Extract the text of all ``

        `` elements from a HTML response body, - returning a list of unicode strings:: - - x.select("h1").extract() # Includes the h1 tag - x.select("h1::text").extract() # Only text inside the h1 tag - -3. Iterate over all ``

        `` tags and print their class attribute:: - - for node in x.select("p"): - ... print node.select("::attr(class)").extract() - -XmlCSSSelector objects ----------------------- - -.. class:: XmlCSSSelector(response) - - A specialized class for working with XML content using `CSS`_ selectors. - -XmlCSSSelector examples -~~~~~~~~~~~~~~~~~~~~~~~ - -Here's a couple of :class:`XmlCSSSelector` examples to illustrate several -concepts. In both cases we assume there is already an :class:`XmlCSSSelector` -instantiated with a :class:`~scrapy.http.XmlResponse` object like this:: - - x = XmlCSSSelector(xml_response) - -1. Select all ```` elements from a XML response body, returning a list - of :class:`XmlCSSSelector` objects:: - - x.select("product") - -Note that querying xml namespaces with CSS selectors doesn't work, if you are -interesting in this feature consider contributing to `cssselect`_ project. - -.. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index daa8d83d0..589142aa6 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -9,10 +9,10 @@ scraping code very quickly, without having to run the spider. It's meant to be used for testing data extraction code, but you can actually use it for testing any kind of code as it is also a regular Python shell. -The shell is used for testing XPath expressions and see how they work and what -data they extract from the web pages you're trying to scrape. It allows you to -interactively test your XPaths while you're writing your spider, without having -to run the spider to test every change. +The shell is used for testing XPath or CSS expressions and see how they work +and what data they extract from the web pages you're trying to scrape. It +allows you to interactively test your expressions while you're writing your +spider, without having to run the spider to test every change. Once you get familiarized with the Scrapy shell, you'll see that it's an invaluable tool for developing and debugging your spiders. @@ -66,7 +66,7 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.XPathSelector` objects (for both HTML and XML +:class:`~scrapy.selector.Selector` objects (for both HTML and XML content). Those objects are: @@ -83,10 +83,7 @@ Those objects are: * ``response`` - a :class:`~scrapy.http.Response` object containing the last fetched page - * ``hxs`` - a :class:`~scrapy.selector.HtmlXPathSelector` object constructed - with the last response fetched - - * ``xxs`` - a :class:`~scrapy.selector.XmlXPathSelector` object constructed + * ``ss`` - a :class:`~scrapy.selector.Selector` object constructed with the last response fetched * ``settings`` - the current :ref:`Scrapy settings ` @@ -114,13 +111,12 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available objects - [s] hxs + [s] ss [s] item Item() [s] request [s] response [s] settings [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Prints this help. [s] fetch(req_or_url) Fetch a new request or URL and update objects @@ -130,24 +126,23 @@ all start with the ``[s]`` prefix):: After that, we can star playing with the objects:: - >>> hxs.select("//h2/text()").extract()[0] + >>> ss.xpath("//h2/text()").extract()[0] u'Welcome to Scrapy' >>> fetch("http://slashdot.org") [s] Available Scrapy objects: - [s] hxs + [s] ss [s] item JobItem() [s] request [s] response <200 http://slashdot.org> [s] settings [s] spider - [s] xxs [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> hxs.select("//h2/text()").extract() + >>> ss.xpath("//h2/text()").extract() [u'News for nerds, stuff that matters'] >>> request = request.replace(method="POST") @@ -185,7 +180,7 @@ When you run the spider, you will get something similar to this:: 2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled (referer: ) 2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled (referer: ) [s] Available objects - [s] hxs + [s] ss ... >>> response.url @@ -193,7 +188,7 @@ When you run the spider, you will get something similar to this:: Then, you can check if the extraction code is working:: - >>> hxs.select('//h1') + >>> ss.xpath('//h1') [] Nope, it doesn't. So you can open the response in your web browser and see if diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 82e96df1e..28393dfe2 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -216,7 +216,7 @@ Let's see an example:: Another example returning multiples Requests and Items from a single callback:: - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from scrapy.spider import BaseSpider from scrapy.http import Request from myproject.items import MyItem @@ -231,11 +231,11 @@ Another example returning multiples Requests and Items from a single callback:: ] def parse(self, response): - hxs = HtmlXPathSelector(response) - for h3 in hxs.select('//h3').extract(): + ss = Selector(response) + for h3 in ss.xpath('//h3').extract(): yield MyItem(title=h3) - for url in hxs.select('//a/@href').extract(): + for url in ss.xpath('//a/@href').extract(): yield Request(url, callback=self.parse) .. module:: scrapy.contrib.spiders @@ -314,7 +314,7 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor - from scrapy.selector import HtmlXPathSelector + from scrapy.selector import Selector from scrapy.item import Item class MySpider(CrawlSpider): @@ -334,11 +334,11 @@ Let's now take a look at an example CrawlSpider with rules:: def parse_item(self, response): self.log('Hi, this is an item page! %s' % response.url) - hxs = HtmlXPathSelector(response) + ss = Selector(response) item = Item() - item['id'] = hxs.select('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = hxs.select('//td[@id="item_name"]/text()').extract() - item['description'] = hxs.select('//td[@id="item_description"]/text()').extract() + item['id'] = ss.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') + item['name'] = ss.xpath('//td[@id="item_name"]/text()').extract() + item['description'] = ss.xpath('//td[@id="item_description"]/text()').extract() return item @@ -366,15 +366,15 @@ XMLFeedSpider A string which defines the iterator to use. It can be either: - - ``'iternodes'`` - a fast iterator based on regular expressions + - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses HtmlXPathSelector. Keep in mind - this uses DOM parsing and must load all DOM in memory which could be a - problem for big feeds + - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + Keep in mind this uses DOM parsing and must load all DOM in memory + which could be a problem for big feeds - - ``'xml'`` - an iterator which uses XmlXPathSelector. Keep in mind - this uses DOM parsing and must load all DOM in memory which could be a - problem for big feeds + - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + Keep in mind this uses DOM parsing and must load all DOM in memory + which could be a problem for big feeds It defaults to: ``'iternodes'``. @@ -390,7 +390,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.XPathSelector.register_namespace` method. + :meth:`~scrapy.selector.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -416,9 +416,10 @@ XMLFeedSpider .. method:: parse_node(response, selector) This method is called for the nodes matching the provided tag name - (``itertag``). Receives the response and an XPathSelector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either a :class:`~scrapy.item.Item` object, a + (``itertag``). Receives the response and an + :class:`~scrapy.selector.Selector` for each node. Overriding this + method is mandatory. Otherwise, you spider won't work. This method + must return either a :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request` object, or an iterable containing any of them. @@ -451,9 +452,9 @@ These spiders are pretty easy to use, let's have a look at one example:: log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) item = Item() - item['id'] = node.select('@id').extract() - item['name'] = node.select('name').extract() - item['description'] = node.select('description').extract() + item['id'] = node.xpath('@id').extract() + item['name'] = node.xpath('name').extract() + item['description'] = node.xpath('description').extract() return item Basically what we did up there was to create a spider that downloads a feed from diff --git a/sep/sep-020.rst b/sep/sep-020.rst index e1d563ba7..7b2c043b7 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -91,9 +91,9 @@ This is a working code sample that covers just the basics. """ Pull the text label out of selected markup :param entity: Found markup - :type entity: HtmlXPathSelector + :type entity: Selector """ - label = ' '.join(entity.select('.//text()').extract()) + label = ' '.join(entity.xpath('.//text()').extract()) label = label.encode('ascii', 'xmlcharrefreplace') if label else '' label = label.strip(' ') if ' ' in label else label label = label.strip(':') if ':' in label else label @@ -108,7 +108,7 @@ This is a working code sample that covers just the basics. :return: The list of selectors :rtype: list """ - return self.selector.select(self.base_xpath + xpath) + return self.selector.xpath(self.base_xpath + xpath) def parse_dl(self, xpath=u'//dl'): """ Look for the specified definition list pattern and store all found @@ -120,7 +120,7 @@ This is a working code sample that covers just the basics. for term in self._get_entities(xpath + '/dt'): label = self._get_label(term) if label and label not in self.ignore: - value = term.select('following-sibling::dd[1]//text()') + value = term.xpath('following-sibling::dd[1]//text()') if value: self.add_value(label, value.extract(), MapCompose(lambda v: v.strip())) From ab9462a251ce084698e3751d5131ab9ce267ab0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 16:37:14 -0200 Subject: [PATCH 15/19] remove more references to libxml2 --- docs/topics/leaks.rst | 1 - scrapy/tests/test_selector_lxmldocument.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index de9bf645e..c60acb47d 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -89,7 +89,6 @@ subclasses): * ``scrapy.item.Item`` * ``scrapy.selector.Selector`` * ``scrapy.spider.BaseSpider`` -* ``scrapy.selector.document.Libxml2Document`` A real example -------------- diff --git a/scrapy/tests/test_selector_lxmldocument.py b/scrapy/tests/test_selector_lxmldocument.py index e544613d5..7dab1d4b1 100644 --- a/scrapy/tests/test_selector_lxmldocument.py +++ b/scrapy/tests/test_selector_lxmldocument.py @@ -3,7 +3,7 @@ from scrapy.selector.lxmldocument import LxmlDocument from scrapy.http import TextResponse, HtmlResponse -class Libxml2DocumentTest(unittest.TestCase): +class LxmlDocumentTest(unittest.TestCase): def test_caching(self): r1 = HtmlResponse('http://www.example.com', body='') From 28999590fa7e383c8ba152c1c843182adc903fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 Oct 2013 16:41:04 -0200 Subject: [PATCH 16/19] update release notes --- docs/news.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/news.rst b/docs/news.rst index dd4f571b6..0f632295a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,6 +11,7 @@ Release notes - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) - Dropped libxml2 selectors backend - Dropped support for multiple selectors backends, sticking to lxml only +- Selector Unified API with support for CSS expressions (:issue:`395` and :issue:`426`) 0.18.4 (released 2013-10-10) ---------------------------- From 1abb1af0c609447e487775700bef05d0b4149101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 Oct 2013 10:13:43 -0200 Subject: [PATCH 17/19] fix typos and wording on selector's introduction --- docs/intro/tutorial.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5068cd8c1..c147d0497 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -208,7 +208,7 @@ XPath expressions are indeed much more powerful. To learn more about XPath we recommend `this XPath tutorial `_. For working with XPaths, Scrapy provides a :class:`~scrapy.selector.Selector` -class, it must be instantiated with a :class:`~scrapy.http.HtmlResponse` or +class, it is instantiated with a :class:`~scrapy.http.HtmlResponse` or :class:`~scrapy.http.XmlResponse` object as first argument. You can see selectors as objects that represent nodes in the document @@ -271,8 +271,9 @@ After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body of the response, or you can type ``response.headers`` to see its headers. -The shell also pre-instantiate a selector named ``ss``, it automatically choice -the best parsing rules (XML vs HTML) based on response's type. +The shell also pre-instantiate a selector for this response in variable ``ss``, +the selector automatically chooses the best parsing rules (XML vs HTML) based +on response's type. So let's try it:: From 155ea08ea1c50f347c21cb4635b36dd47141f4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 Oct 2013 15:58:36 -0200 Subject: [PATCH 18/19] use `sel` name for Selector's instances in docs, internals and shell --- docs/intro/overview.rst | 8 +-- docs/intro/tutorial.rst | 32 ++++++------ docs/topics/firebug.rst | 4 +- docs/topics/selectors.rst | 34 ++++++------ docs/topics/shell.rst | 14 ++--- docs/topics/spiders.rst | 14 ++--- scrapy/contrib/linkextractors/sgml.py | 4 +- scrapy/shell.py | 4 +- scrapy/tests/test_command_shell.py | 2 +- scrapy/tests/test_selector.py | 58 ++++++++++----------- scrapy/tests/test_selector_csstranslator.py | 16 +++--- 11 files changed, 95 insertions(+), 95 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 2138e76f8..588f33755 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -143,12 +143,12 @@ Finally, here's the spider code:: rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] def parse_torrent(self, response): - ss = Selector(response) + sel = Selector(response) torrent = TorrentItem() torrent['url'] = response.url - torrent['name'] = ss.xpath("//h1/text()").extract() - torrent['description'] = ss.xpath("//div[@id='description']").extract() - torrent['size'] = ss.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() + torrent['name'] = sel.xpath("//h1/text()").extract() + torrent['description'] = sel.xpath("//div[@id='description']").extract() + torrent['size'] = sel.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() return torrent For brevity's sake, we intentionally left out the import statements. The diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c147d0497..fa1909479 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -255,7 +255,7 @@ This is what the shell looks like:: [s] Available Scrapy objects: [s] 2010-08-19 21:45:59-0300 [default] INFO: Spider closed (finished) - [s] ss + [s] sel [s] item Item() [s] request [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> @@ -271,25 +271,25 @@ After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body of the response, or you can type ``response.headers`` to see its headers. -The shell also pre-instantiate a selector for this response in variable ``ss``, +The shell also pre-instantiate a selector for this response in variable ``sel``, the selector automatically chooses the best parsing rules (XML vs HTML) based on response's type. So let's try it:: - In [1]: ss.xpath('//title') + In [1]: sel.xpath('//title') Out[1]: [] - In [2]: ss.xpath('//title').extract() + In [2]: sel.xpath('//title').extract() Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: ss.xpath('//title/text()') + In [3]: sel.xpath('//title/text()') Out[3]: [] - In [4]: ss.xpath('//title/text()').extract() + In [4]: sel.xpath('//title/text()').extract() Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: ss.xpath('//title/text()').re('(\w+):') + In [5]: sel.xpath('//title/text()').re('(\w+):') Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data @@ -309,25 +309,25 @@ is inside a ``

          `` element, in fact the *second* ``
            `` element. So we can select each ``
          • `` element belonging to the sites list with this code:: - ss.xpath('//ul/li') + sel.xpath('//ul/li') And from them, the sites descriptions:: - ss.xpath('//ul/li/text()').extract() + sel.xpath('//ul/li/text()').extract() The sites titles:: - ss.xpath('//ul/li/a/text()').extract() + sel.xpath('//ul/li/a/text()').extract() And the sites links:: - ss.xpath('//ul/li/a/@href').extract() + sel.xpath('//ul/li/a/@href').extract() As we said before, each ``.xpath()`` call returns a list of selectors, so we can concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use that property here, so:: - sites = ss.xpath('//ul/li') + sites = sel.xpath('//ul/li') for site in sites: title = site.xpath('a/text()').extract() link = site.xpath('a/@href').extract() @@ -355,8 +355,8 @@ Let's add this code to our spider:: ] def parse(self, response): - ss = Selector(response) - sites = ss.xpath('//ul/li') + sel = Selector(response) + sites = sel.xpath('//ul/li') for site in sites: title = site.xpath('a/text()').extract() link = site.xpath('a/@href').extract() @@ -398,8 +398,8 @@ scraped so far, the final code for our Spider would be like this:: ] def parse(self, response): - ss = Selector(response) - sites = ss.xpath('//ul/li') + sel = Selector(response) + sites = sel.xpath('//ul/li') items = [] for site in sites: item = DmozItem() diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 839701a32..5aa03a65e 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -146,10 +146,10 @@ that have that grey colour of the links, Finally, we can write our ``parse_category()`` method:: def parse_category(self, response): - ss = Selector(response) + sel = Selector(response) # The path to website links in directory page - links = ss.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + links = sel.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') for link in links: item = DirectoryItem() diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b9b185d4c..fd986d0cd 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -62,13 +62,13 @@ body is what they're going to be "selecting":: class MySpider(BaseSpider): # ... def parse(self, response): - ss = Selector(response) + sel = Selector(response) # Using XPath query - print ss.xpath('//p') + print sel.xpath('//p') # Using CSS query - print ss.css('p') + print sel.css('p') # Nesting queries - print ss.xpath('//div[@foo="bar"]').css('span#bold') + print sel.xpath('//div[@foo="bar"]').css('span#bold') Using selectors @@ -94,7 +94,7 @@ First, let's open the shell:: scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have a selector already instantiated and -ready to use in ``ss`` shell variable. +ready to use in ``sel`` shell variable. Since we're dealing with HTML, the selector will automatically use an HTML parser. @@ -104,7 +104,7 @@ So, by looking at the :ref:`HTML code ` of that page, let's construct an XPath (using an HTML selector) for selecting the text inside the title tag:: - >>> ss.xpath('//title/text()') + >>> sel.xpath('//title/text()') [] As you can see, the ``.xpath()`` method returns an @@ -114,45 +114,45 @@ selectors. This API can be used quickly for extracting nested data. To actually extract the textual data, you must call the selector ``.extract()`` method, as follows:: - >>> ss.xpath('//title/text()').extract() + >>> sel.xpath('//title/text()').extract() [u'Example website'] Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: - >>> ss.css('title::text').extract() + >>> sel.css('title::text').extract() [u'Example website'] Now we're going to get the base URL and some image links:: - >>> ss.xpath('//base/@href').extract() + >>> sel.xpath('//base/@href').extract() [u'http://example.com/'] - >>> ss.css('base::attr(href)').extract() + >>> sel.css('base::attr(href)').extract() [u'http://example.com/'] - >>> ss.xpath('//a[contains(@href, "image")]/@href').extract() + >>> sel.xpath('//a[contains(@href, "image")]/@href').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> ss.css('a[href*=image]::attr(href)').extract() + >>> sel.css('a[href*=image]::attr(href)').extract() [u'image1.html', u'image2.html', u'image3.html', u'image4.html', u'image5.html'] - >>> ss.xpath('//a[contains(@href, "image")]/img/@src').extract() + >>> sel.xpath('//a[contains(@href, "image")]/img/@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> ss.css('a[href*=image] img::attr(src)').extract() + >>> sel.css('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -168,7 +168,7 @@ The selection methods (``.xpath()`` or ``.css()``) returns a list of selectors of the same type, so you can call the selection methods for those selectors too. Here's an example:: - >>> links = ss.xpath('//a[contains(@href, "image")]') + >>> links = sel.xpath('//a[contains(@href, "image")]') >>> links.extract() [u'Name: My image 1
            ', u'Name: My image 2
            ', @@ -197,7 +197,7 @@ can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code ` above:: - >>> ss.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + >>> sel.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') [u'My image 1', u'My image 2', u'My image 3', @@ -216,7 +216,7 @@ with ``/``, that XPath will be absolute to the document and not relative to the For example, suppose you want to extract all ``

            `` elements inside ``

            `` elements. First, you would get all ``
            `` elements:: - >>> divs = ss.xpath('//div') + >>> divs = sel.xpath('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``

            `` elements from the document, not only those diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 589142aa6..561e299f4 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -83,7 +83,7 @@ Those objects are: * ``response`` - a :class:`~scrapy.http.Response` object containing the last fetched page - * ``ss`` - a :class:`~scrapy.selector.Selector` object constructed + * ``sel`` - a :class:`~scrapy.selector.Selector` object constructed with the last response fetched * ``settings`` - the current :ref:`Scrapy settings ` @@ -111,7 +111,7 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available objects - [s] ss + [s] sel [s] item Item() [s] request [s] response @@ -126,12 +126,12 @@ all start with the ``[s]`` prefix):: After that, we can star playing with the objects:: - >>> ss.xpath("//h2/text()").extract()[0] + >>> sel.xpath("//h2/text()").extract()[0] u'Welcome to Scrapy' >>> fetch("http://slashdot.org") [s] Available Scrapy objects: - [s] ss + [s] sel [s] item JobItem() [s] request [s] response <200 http://slashdot.org> @@ -142,7 +142,7 @@ After that, we can star playing with the objects:: [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> ss.xpath("//h2/text()").extract() + >>> sel.xpath("//h2/text()").extract() [u'News for nerds, stuff that matters'] >>> request = request.replace(method="POST") @@ -180,7 +180,7 @@ When you run the spider, you will get something similar to this:: 2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled (referer: ) 2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled (referer: ) [s] Available objects - [s] ss + [s] sel ... >>> response.url @@ -188,7 +188,7 @@ When you run the spider, you will get something similar to this:: Then, you can check if the extraction code is working:: - >>> ss.xpath('//h1') + >>> sel.xpath('//h1') [] Nope, it doesn't. So you can open the response in your web browser and see if diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 28393dfe2..dfbdcea18 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -231,11 +231,11 @@ Another example returning multiples Requests and Items from a single callback:: ] def parse(self, response): - ss = Selector(response) - for h3 in ss.xpath('//h3').extract(): + sel = Selector(response) + for h3 in sel.xpath('//h3').extract(): yield MyItem(title=h3) - for url in ss.xpath('//a/@href').extract(): + for url in sel.xpath('//a/@href').extract(): yield Request(url, callback=self.parse) .. module:: scrapy.contrib.spiders @@ -334,11 +334,11 @@ Let's now take a look at an example CrawlSpider with rules:: def parse_item(self, response): self.log('Hi, this is an item page! %s' % response.url) - ss = Selector(response) + sel = Selector(response) item = Item() - item['id'] = ss.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = ss.xpath('//td[@id="item_name"]/text()').extract() - item['description'] = ss.xpath('//td[@id="item_description"]/text()').extract() + item['id'] = sel.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') + item['name'] = sel.xpath('//td[@id="item_name"]/text()').extract() + item['description'] = sel.xpath('//td[@id="item_description"]/text()').extract() return item diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index f0e141a43..d8f6ae4ec 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -116,11 +116,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): base_url = None if self.restrict_xpaths: - ss = Selector(response) + sel = Selector(response) base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths - for f in ss.xpath(x).extract() + for f in sel.xpath(x).extract() ).encode(response.encoding) else: body = response.body diff --git a/scrapy/shell.py b/scrapy/shell.py index 8b6c209cb..8f60b58a5 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -17,7 +17,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.console import start_python_console from scrapy.settings import Settings -from scrapy.http import Request, Response, HtmlResponse, XmlResponse +from scrapy.http import Request, Response from scrapy.exceptions import IgnoreRequest @@ -95,7 +95,7 @@ class Shell(object): self.vars['spider'] = spider self.vars['request'] = request self.vars['response'] = response - self.vars['ss'] = Selector(response) + self.vars['sel'] = Selector(response) if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser diff --git a/scrapy/tests/test_command_shell.py b/scrapy/tests/test_command_shell.py index 8dbaa632f..1fbb4b28a 100644 --- a/scrapy/tests/test_command_shell.py +++ b/scrapy/tests/test_command_shell.py @@ -31,7 +31,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'ss.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'sel.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), 'Works') diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 4149b837c..d8ff6d346 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -16,31 +16,31 @@ class SelectorTestCase(unittest.TestCase): """Simple selector tests""" body = "

            " response = TextResponse(url="http://example.com", body=body) - ss = self.sscls(response) + sel = self.sscls(response) - xl = ss.xpath('//input') + xl = sel.xpath('//input') self.assertEqual(2, len(xl)) for x in xl: assert isinstance(x, self.sscls) - self.assertEqual(ss.xpath('//input').extract(), - [x.extract() for x in ss.xpath('//input')]) + self.assertEqual(sel.xpath('//input').extract(), + [x.extract() for x in sel.xpath('//input')]) - self.assertEqual([x.extract() for x in ss.xpath("//input[@name='a']/@name")], + self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in ss.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(ss.xpath("concat('xpath', 'rules')").extract(), + self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(), [u'xpathrules']) - self.assertEqual([x.extract() for x in ss.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) def test_select_unicode_query(self): body = u"

            " response = TextResponse(url="http://example.com", body=body, encoding='utf8') - ss = self.sscls(response) - self.assertEqual(ss.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1']) + sel = self.sscls(response) + self.assertEqual(sel.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1']) def test_list_elements_type(self): """Test Selector returning the same type in selection methods""" @@ -69,14 +69,14 @@ class SelectorTestCase(unittest.TestCase): def test_flavor_detection(self): text = '

            Hello

            ' - ss = self.sscls(XmlResponse('http://example.com', body=text)) - self.assertEqual(ss.contenttype, 'xml') - self.assertEqual(ss.xpath("//div").extract(), + sel = self.sscls(XmlResponse('http://example.com', body=text)) + self.assertEqual(sel.contenttype, 'xml') + self.assertEqual(sel.xpath("//div").extract(), [u'

            Hello

            ']) - ss = self.sscls(HtmlResponse('http://example.com', body=text)) - self.assertEqual(ss.contenttype, 'html') - self.assertEqual(ss.xpath("//div").extract(), + sel = self.sscls(HtmlResponse('http://example.com', body=text)) + self.assertEqual(sel.contenttype, 'html') + self.assertEqual(sel.xpath("//div").extract(), [u'

            Hello

            ']) def test_nested_selectors(self): @@ -110,13 +110,13 @@ class SelectorTestCase(unittest.TestCase):
            notme

            text

            foo
            ''' - ss = self.sscls(text=body) - self.assertEqual(ss.xpath('//div[@id="1"]').css('span::text').extract(), [u'me']) - self.assertEqual(ss.css('#1').xpath('./span/text()').extract(), [u'me']) + sel = self.sscls(text=body) + self.assertEqual(sel.xpath('//div[@id="1"]').css('span::text').extract(), [u'me']) + self.assertEqual(sel.css('#1').xpath('./span/text()').extract(), [u'me']) def test_dont_strip(self): - hxs = self.sscls(text='
            fff: zzz
            ') - self.assertEqual(hxs.xpath("//text()").extract(), [u'fff: ', u'zzz']) + sel = self.sscls(text='
            fff: zzz
            ') + self.assertEqual(sel.xpath("//text()").extract(), [u'fff: ', u'zzz']) def test_namespaces_simple(self): body = """ @@ -279,10 +279,10 @@ class SelectorTestCase(unittest.TestCase): """ - xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.xpath("//link")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.xpath("//link")), 2) + sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(sel.xpath("//link")), 0) + sel.remove_namespaces() + self.assertEqual(len(sel.xpath("//link")), 2) def test_remove_attributes_namespaces(self): xml = """ @@ -291,10 +291,10 @@ class SelectorTestCase(unittest.TestCase): """ - xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.xpath("//link/@type")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.xpath("//link/@type")), 2) + sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(sel.xpath("//link/@type")), 0) + sel.remove_namespaces() + self.assertEqual(len(sel.xpath("//link/@type")), 2) class DeprecatedXpathSelectorTest(unittest.TestCase): diff --git a/scrapy/tests/test_selector_csstranslator.py b/scrapy/tests/test_selector_csstranslator.py index 45ef91c19..7ef9003aa 100644 --- a/scrapy/tests/test_selector_csstranslator.py +++ b/scrapy/tests/test_selector_csstranslator.py @@ -120,16 +120,16 @@ class CSSSelectorTest(unittest.TestCase): def setUp(self): self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY) - self.ss = self.sscls(self.htmlresponse) + self.sel = self.sscls(self.htmlresponse) def x(self, *a, **kw): - return [v.strip() for v in self.ss.css(*a, **kw).extract() if v.strip()] + return [v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()] def test_selector_simple(self): - for x in self.ss.css('input'): - self.assertTrue(isinstance(x, self.ss.__class__), x) - self.assertEqual(self.ss.css('input').extract(), - [x.extract() for x in self.ss.css('input')]) + for x in self.sel.css('input'): + self.assertTrue(isinstance(x, self.sel.__class__), x) + self.assertEqual(self.sel.css('input').extract(), + [x.extract() for x in self.sel.css('input')]) def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) @@ -147,7 +147,7 @@ class CSSSelectorTest(unittest.TestCase): self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.ss.css('p').css('b::text').extract(), + self.assertEqual(self.sel.css('p').css('b::text').extract(), [u'hi', u'guy']) - self.assertEqual(self.ss.css('div').css('area:last-child').extract(), + self.assertEqual(self.sel.css('div').css('area:last-child').extract(), [u'']) From 14613638099dffd9a1e67edf1ea63e1729d7b097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 16 Oct 2013 17:37:22 -0200 Subject: [PATCH 19/19] Replace `contenttype` references by `type` The type to choose from is the selector type, not the input type. A content-type doesn't make sense in this context. --- docs/topics/selectors.rst | 26 ++++++++++++++++++++----- scrapy/contrib/spiders/feed.py | 4 ++-- scrapy/contrib_exp/iterators.py | 2 +- scrapy/selector/lxmlsel.py | 6 +++--- scrapy/selector/unified.py | 34 ++++++++++++++++----------------- scrapy/tests/test_selector.py | 10 +++++----- scrapy/utils/iterators.py | 2 +- 7 files changed, 49 insertions(+), 35 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index fd986d0cd..efc61a100 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -249,18 +249,34 @@ Built-in Selectors reference .. module:: scrapy.selector :synopsis: Selector class -.. class:: Selector(response, contenttype=None) +.. class:: Selector(response=None, text=None, type=None) - An instance of :class:`Selector` is a wrapper over ``response`` to select + An instance of :class:`Selector` is a wrapper over response to select certain parts of its content. ``response`` is a :class:`~scrapy.http.HtmlResponse` or :class:`~scrapy.http.XmlResponse` object that will be used for selecting and extracting data. - ``contenttype`` tells what parser and selection flavor is used to parse the - response body. It defaults to ``"html"`` for :class:`~scrapy.http.HtmlResponse` - and ``"xml"`` for :class:`~scrapy.http.XmlResponse`. + ``text`` is a unicode string or utf-8 encoded text for cases when a + ``response`` isn't available. Using ``text`` and ``response`` together is + undefined behavior. + + ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default). + + If ``type`` is ``None``, the selector automatically chooses the best type + based on ``response`` type (see below), or defaults to ``"html"`` in case it + is used together with ``text``. + + If ``type`` is ``None`` and a ``response`` is passed, the selector type is + inferred from the response type as follow: + + * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type + * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"html"`` for anything else + + Otherwise, if ``type`` is set, the selector type will be forced and no + detection will occur. .. method:: xpath(query) diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 6d8e0a358..271b3a97a 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -71,11 +71,11 @@ class XMLFeedSpider(BaseSpider): if self.iterator == 'iternodes': nodes = self._iternodes(response) elif self.iterator == 'xml': - selector = Selector(response, contenttype='xml') + selector = Selector(response, type='xml') self._register_namespaces(selector) nodes = selector.xpath('//%s' % self.itertag) elif self.iterator == 'html': - selector = Selector(response, contenttype='html') + selector = Selector(response, type='html') self._register_namespaces(selector) nodes = selector.xpath('//%s' % self.itertag) else: diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py index ecc6265d0..7cf9103fd 100644 --- a/scrapy/contrib_exp/iterators.py +++ b/scrapy/contrib_exp/iterators.py @@ -11,7 +11,7 @@ def xmliter_lxml(obj, nodename, namespace=None): for _, node in iterable: nodetext = etree.tostring(node) node.clear() - xs = Selector(text=nodetext, contenttype='xml') + xs = Selector(text=nodetext, type='xml') if namespace: xs.register_namespace('x', namespace) yield xs.xpath(selxpath)[0] diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index bc655b523..de897d2fd 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -10,7 +10,7 @@ __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', class XPathSelector(Selector): __slots__ = () - default_contenttype = 'html' + _default_type = 'html' def __init__(self, *a, **kw): import warnings @@ -28,12 +28,12 @@ class XPathSelector(Selector): class XmlXPathSelector(XPathSelector): __slots__ = () - default_contenttype = 'xml' + _default_type = 'xml' class HtmlXPathSelector(XPathSelector): __slots__ = () - default_contenttype = 'html' + _default_type = 'html' class XPathSelectorList(SelectorList): diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 77cd9e11f..a316c20cf 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -25,38 +25,37 @@ _ctgroup = { } -def _ct(response, ct): - if ct is None: +def _st(response, st): + if st is None: return 'xml' if isinstance(response, XmlResponse) else 'html' - elif ct in ('xml', 'html'): - return ct + elif st in ('xml', 'html'): + return st else: - raise ValueError('Invalid contenttype: %s' % ct) + raise ValueError('Invalid type: %s' % st) -def _response_from_text(text, ct): - rt = XmlResponse if ct == 'xml' else HtmlResponse +def _response_from_text(text, st): + rt = XmlResponse if st == 'xml' else HtmlResponse return rt(url='about:blank', encoding='utf-8', body=unicode_to_str(text, 'utf-8')) class Selector(object_ref): - __slots__ = ['response', 'text', 'namespaces', 'contenttype', '_expr', '_root', + __slots__ = ['response', 'text', 'namespaces', 'type', '_expr', '_root', '__weakref__', '_parser', '_csstranslator', '_tostring_method'] - default_contenttype = None + _default_type = None - def __init__(self, response=None, text=None, namespaces=None, contenttype=None, + def __init__(self, response=None, text=None, type=None, namespaces=None, _root=None, _expr=None): - - self.contenttype = ct = _ct(response, contenttype or self.default_contenttype) - self._parser = _ctgroup[ct]['_parser'] - self._csstranslator = _ctgroup[ct]['_csstranslator'] - self._tostring_method = _ctgroup[ct]['_tostring_method'] + self.type = st = _st(response, type or self._default_type) + self._parser = _ctgroup[st]['_parser'] + self._csstranslator = _ctgroup[st]['_csstranslator'] + self._tostring_method = _ctgroup[st]['_tostring_method'] if text is not None: - response = _response_from_text(text, ct) + response = _response_from_text(text, st) if response is not None: _root = LxmlDocument(response, self._parser) @@ -82,7 +81,7 @@ class Selector(object_ref): result = [self.__class__(_root=x, _expr=query, namespaces=self.namespaces, - contenttype=self.contenttype) + type=self.type) for x in result] return SelectorList(result) @@ -169,4 +168,3 @@ class SelectorList(list): @deprecated(use_instead='.xpath()') def select(self, xpath): return self.xpath(xpath) - diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index d8ff6d346..93d0b9532 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -59,23 +59,23 @@ class SelectorTestCase(unittest.TestCase): """Test that XML and HTML Selector's behave differently""" # some text which is parsed differently by XML and HTML flavors text = '

            Hello

            ' - hs = self.sscls(text=text, contenttype='html') + hs = self.sscls(text=text, type='html') self.assertEqual(hs.xpath("//div").extract(), [u'

            Hello

            ']) - xs = self.sscls(text=text, contenttype='xml') + xs = self.sscls(text=text, type='xml') self.assertEqual(xs.xpath("//div").extract(), [u'

            Hello

            ']) def test_flavor_detection(self): text = '

            Hello

            ' sel = self.sscls(XmlResponse('http://example.com', body=text)) - self.assertEqual(sel.contenttype, 'xml') + self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").extract(), [u'

            Hello

            ']) sel = self.sscls(HtmlResponse('http://example.com', body=text)) - self.assertEqual(sel.contenttype, 'html') + self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").extract(), [u'

            Hello

            ']) @@ -183,7 +183,7 @@ class SelectorTestCase(unittest.TestCase): def test_selector_over_text(self): hs = self.sscls(text='lala') self.assertEqual(hs.extract(), u'lala') - xs = self.sscls(text='lala', contenttype='xml') + xs = self.sscls(text='lala', type='xml') self.assertEqual(xs.extract(), u'lala') self.assertEqual(xs.xpath('.').extract(), [u'lala']) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 67f9946e6..18fa56de4 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -29,7 +29,7 @@ def xmliter(obj, nodename): r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, contenttype='xml').xpath('//' + nodename)[0] + yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] def csviter(obj, delimiter=None, headers=None, encoding=None):