diff --git a/.gitignore b/.gitignore
index ff6e2ea65..7392ed31e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
.pytest_cache/
.coverage.*
.cache/
+.pytest_cache/
# Windows
Thumbs.db
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6f1c2c43f..9d7c94d39 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.xpath('span/small/text()').extract_first(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.xpath('span/small/text()').get(),
}
- next_page = response.css('li.next a::attr("href")').extract_first()
+ next_page = response.css('li.next a::attr("href")').get()
if next_page is not None:
yield response.follow(next_page, self.parse)
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fa6dc274d..ad17ef096 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -254,7 +254,7 @@ data.
To extract the text from the title above, you can do::
- >>> response.css('title::text').extract()
+ >>> response.css('title::text').getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
@@ -262,32 +262,33 @@ CSS query, to mean we want to select only the text elements directly inside
```` element. If we don't specify ``::text``, we'd get the full title
element, including its tags::
- >>> response.css('title').extract()
+ >>> response.css('title').getall()
['<title>Quotes to Scrape']
-The other thing is that the result of calling ``.extract()`` is a list, because
-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When
-you know you just want the first result, as in this case, you can do::
+The other thing is that the result of calling ``.getall()`` is a list: it is
+possible that a selector returns more than one result, so we extract them all.
+When you know you just want the first result, as in this case, you can do::
- >>> response.css('title::text').extract_first()
+ >>> response.css('title::text').get()
'Quotes to Scrape'
As an alternative, you could've written::
- >>> response.css('title::text')[0].extract()
+ >>> response.css('title::text')[0].get()
'Quotes to Scrape'
-However, using ``.extract_first()`` avoids an ``IndexError`` and returns
-``None`` when it doesn't find any element matching the selection.
+However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
+instance avoids an ``IndexError`` and returns ``None`` when it doesn't
+find any element matching the selection.
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
to be scraped, you can at least get **some** data.
-Besides the :meth:`~scrapy.selector.Selector.extract` and
-:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
-expressions`::
+Besides the :meth:`~scrapy.selector.SelectorList.getall` and
+:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
+expressions`_::
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
@@ -298,7 +299,8 @@ expressions`::
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
+You can use your browser developer tools to inspect the HTML and come up
+with a selector (see section about :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
@@ -314,7 +316,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
>>> response.xpath('//title')
[]
- >>> response.xpath('//title/text()').extract_first()
+ >>> response.xpath('//title/text()').get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
@@ -383,17 +385,17 @@ variable, so that we can run our CSS selectors directly on a particular quote::
Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created::
- >>> title = quote.css("span.text::text").extract_first()
+ >>> title = quote.css("span.text::text").get()
>>> title
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
- >>> author = quote.css("small.author::text").extract_first()
+ >>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
-Given that the tags are a list of strings, we can use the ``.extract()`` method
+Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them::
- >>> tags = quote.css("div.tags a.tag::text").extract()
+ >>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
@@ -401,9 +403,9 @@ Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary::
>>> for quote in response.css("div.quote"):
- ... text = quote.css("span.text::text").extract_first()
- ... author = quote.css("small.author::text").extract_first()
- ... tags = quote.css("div.tags a.tag::text").extract()
+ ... text = quote.css("span.text::text").get()
+ ... author = quote.css("small.author::text").get()
+ ... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
@@ -434,9 +436,9 @@ in the callback, as you can see below::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
If you run this spider, it will output the extracted data with the log::
@@ -508,16 +510,22 @@ markup:
We can try extracting it in the shell::
- >>> response.css('li.next a').extract_first()
+ >>> response.css('li.next a').get()
'Next →'
This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that let's you select the attribute contents,
like this::
- >>> response.css('li.next a::attr(href)').extract_first()
+ >>> response.css('li.next a::attr(href)').get()
'/page/2/'
+There is also an ``attrib`` property available
+(see :ref:`selecting-attributes` for more)::
+
+ >>> response.css('li.next a').attrib['href']
+ '/page/2'
+
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
@@ -533,12 +541,12 @@ page, extracting data from it::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@@ -584,12 +592,12 @@ As a shortcut for creating Request objects you can use
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('span small::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('span small::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
@@ -641,7 +649,7 @@ this time for scraping author information::
def parse_author(self, response):
def extract_with_css(query):
- return response.css(query).extract_first().strip()
+ return response.css(query).get(default='').strip()
yield {
'name': extract_with_css('h3.author-title::text'),
@@ -710,11 +718,11 @@ with a specific tag, building the URL based on the argument::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, self.parse)
@@ -738,4 +746,3 @@ modeling the scraped data. If you prefer to play with an example project, check
the :ref:`intro-examples` section.
.. _JSON: https://en.wikipedia.org/wiki/JSON
-.. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3088017cb..ef9c45196 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -458,9 +458,9 @@ Usage example::
>>> STATUS DEPTH LEVEL 1 <<<
# Scraped Items ------------------------------------------------------------
- [{'name': u'Example item',
- 'category': u'Furniture',
- 'length': u'12 cm'}]
+ [{'name': 'Example item',
+ 'category': 'Furniture',
+ 'length': '12 cm'}]
# Requests -----------------------------------------------------------------
[]
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4423bbda2..ae44aecd3 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -86,7 +86,7 @@ Creating items
::
>>> product = Product(name='Desktop PC', price=1000)
- >>> print product
+ >>> print(product)
Product(name='Desktop PC', price=1000)
Getting field values
@@ -161,11 +161,11 @@ Other common tasks
Copying items::
>>> product2 = Product(product)
- >>> print product2
+ >>> print(product2)
Product(name='Desktop PC', price=1000)
>>> product3 = product2.copy()
- >>> print product3
+ >>> print(product3)
Product(name='Desktop PC', price=1000)
Creating dicts from items::
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 06c7fff3d..8e1574376 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -84,7 +84,7 @@ So, for example, this won't work::
return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
def other_callback(self, response, somearg):
- print "the argument passed is:", somearg
+ print("the argument passed is: %s" % somearg)
But this will::
@@ -94,7 +94,7 @@ But this will::
def other_callback(self, response):
somearg = response.meta['somearg']
- print "the argument passed is:", somearg
+ print("the argument passed is: %s" % somearg)
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a895b535c..f3b6aa4a1 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -678,10 +678,10 @@ Here is a list of all built-in processors:
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
- u'one two three'
+ 'one two three'
>>> proc = Join('
')
>>> proc(['one', 'two', 'three'])
- u'one
two
three'
+ 'one
two
three'
.. class:: Compose(\*functions, \**default_loader_context)
@@ -744,9 +744,9 @@ Here is a list of all built-in processors:
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
- >>> proc = MapCompose(filter_world, unicode.upper)
- >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
- [u'HELLO, u'THIS', u'IS', u'SCRAPY']
+ >>> proc = MapCompose(filter_world, str.upper)
+ >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+ ['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
constructor keyword arguments are used as default context values. See
@@ -772,7 +772,7 @@ Here is a list of all built-in processors:
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
- u'bar'
+ 'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
- [u'bar']
+ ['bar']
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ac40c3cc..9dced7473 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -6,7 +6,7 @@ Selectors
When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
-achieve this:
+achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
@@ -25,16 +25,14 @@ either by `XPath`_ or `CSS`_ expressions.
used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
defines selectors to associate those styles with specific HTML elements.
-Scrapy selectors are built over the `lxml`_ library, which means they're very
-similar in speed and parsing accuracy.
+.. note::
+ Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of
+ this wrapper is to provide better integration with Scrapy Response objects.
-This page explains how selectors work and describes their API which is very
-small and simple, unlike the `lxml`_ API which is much bigger because the
-`lxml`_ library can be used for many other tasks, besides selecting markup
-documents.
-
-For a complete reference of the selectors API see
-:ref:`Selector reference `
+ `parsel`_ is a stand-alone web scraping library which can be used without
+ Scrapy. It uses `lxml`_ library under the hood, and implements an
+ easy API on top of lxml API. It means Scrapy selectors are very similar
+ in speed and parsing accuracy to lxml.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
@@ -42,7 +40,7 @@ For a complete reference of the selectors API see
.. _cssselect: https://pypi.python.org/pypi/cssselect/
.. _XPath: https://www.w3.org/TR/xpath
.. _CSS: https://www.w3.org/TR/selectors
-
+.. _parsel: https://parsel.readthedocs.io/
Using selectors
===============
@@ -52,32 +50,48 @@ Constructing selectors
.. highlight:: python
+Response objects expose a :class:`~scrapy.selector.Selector` instance
+on ``.selector`` attribute::
+
+ >>> response.selector.xpath('//span/text()').get()
+ 'good'
+
+Querying responses using XPath and CSS is so common that responses include two
+more shortcuts: ``response.xpath()`` and ``response.css()``::
+
+ >>> response.xpath('//span/text()').get()
+ 'good'
+ >>> response.css('span::text').get()
+ 'good'
+
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing **text** or :class:`~scrapy.http.TextResponse`
-object. It automatically chooses the best parsing rules (XML vs HTML) based on
-input type::
+constructed by passing either :class:`~scrapy.http.TextResponse` object or
+markup as an unicode string (in ``text`` argument).
+Usually there is no need to construct Scrapy selectors manually:
+``response`` object is available in Spider callbacks, so in most cases
+it is more convenient to use ``response.css()`` and ``response.xpath()``
+shortcuts. By using ``response.selector`` or one of these shortcuts
+you can also ensure the response body is parsed only once.
+
+But if required, it is possible to use ``Selector`` directly.
+Constructing from text::
+
+ >>> from scrapy.selector import Selector
+ >>> body = 'good'
+ >>> Selector(text=body).xpath('//span/text()').get()
+ 'good'
+
+Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
+:class:`~scrapy.http.TextResponse` subclasses::
>>> from scrapy.selector import Selector
>>> from scrapy.http import HtmlResponse
-
-Constructing from text::
-
- >>> body = 'good'
- >>> Selector(text=body).xpath('//span/text()').extract()
- [u'good']
-
-Constructing from response::
-
>>> response = HtmlResponse(url='http://example.com', body=body)
- >>> Selector(response=response).xpath('//span/text()').extract()
- [u'good']
-
-For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible::
-
- >>> response.selector.xpath('//span/text()').extract()
- [u'good']
+ >>> Selector(response=response).xpath('//span/text()').get()
+ 'good'
+``Selector`` automatically chooses the best parsing rules
+(XML vs HTML) based on input type.
Using selectors
---------------
@@ -90,7 +104,7 @@ documentation server:
.. _topics-selectors-htmlcode:
-Here's its HTML code:
+For the sake of completeness, here's its full HTML code:
.. literalinclude:: ../_static/selectors-sample1.html
:language: html
@@ -111,90 +125,191 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
So, by looking at the :ref:`HTML code ` of that
page, let's construct an XPath for selecting the text inside the title tag::
- >>> response.selector.xpath('//title/text()')
- []
-
-Querying responses using XPath and CSS is so common that responses include two
-convenience shortcuts: ``response.xpath()`` and ``response.css()``::
-
>>> response.xpath('//title/text()')
- []
- >>> response.css('title::text')
- []
+ []
+
+To actually extract the textual data, you must call the selector ``.get()``
+or ``.getall()`` methods, as follows::
+
+ >>> response.xpath('//title/text()').getall()
+ ['Example website']
+ >>> response.xpath('//title/text()').get()
+ 'Example website'
+
+``.get()`` always returns a single result; if there are several matches,
+content of a first match is returned; if there are no matches, None
+is returned. ``.getall()`` returns a list with all results.
+
+Notice that CSS selectors can select text or attribute nodes using CSS3
+pseudo-elements::
+
+ >>> response.css('title::text').get()
+ 'Example website'
As you can see, ``.xpath()`` and ``.css()`` methods return a
:class:`~scrapy.selector.SelectorList` instance, which is a list of new
selectors. This API can be used for quickly selecting nested data::
- >>> response.css('img').xpath('@src').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+ >>> response.css('img').xpath('@src').getall()
+ ['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
-To actually extract the textual data, you must call the selector ``.extract()``
-method, as follows::
+If you want to extract only the first matched element, you can call the
+selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
+previous Scrapy versions)::
- >>> response.xpath('//title/text()').extract()
- [u'Example website']
+ >>> response.xpath('//div[@id="images"]/a/text()').get()
+ 'Name: My image 1 '
-If you want to extract only first matched element, you can call the selector ``.extract_first()``
+It returns ``None`` if no element was found::
- >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
- u'Name: My image 1 '
-
-It returns ``None`` if no element was found:
-
- >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
+ >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
True
-A default return value can be provided as an argument, to be used instead of ``None``:
+A default return value can be provided as an argument, to be used instead
+of ``None``:
- >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+ >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
'not-found'
-Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
- >>> response.css('title::text').extract()
- [u'Example website']
+ >>> [img.attrib['src'] for img in response.css('img')]
+ ['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
+
+As a shortcut, ``.attrib`` is also available on SelectorList directly;
+it returns attributes for the first matching element::
+
+ >>> response.css('img').attrib['src']
+ 'image1_thumb.jpg'
+
+This is most useful when only a single result is expected, e.g. when selecting
+by id, or selecting unique elements on a web page::
+
+ >>> response.css('base').attrib['href']
+ 'http://example.com/'
Now we're going to get the base URL and some image links::
- >>> response.xpath('//base/@href').extract()
- [u'http://example.com/']
+ >>> response.xpath('//base/@href').get()
+ 'http://example.com/'
- >>> response.css('base::attr(href)').extract()
- [u'http://example.com/']
+ >>> response.css('base::attr(href)').get()
+ 'http://example.com/'
- >>> response.xpath('//a[contains(@href, "image")]/@href').extract()
- [u'image1.html',
- u'image2.html',
- u'image3.html',
- u'image4.html',
- u'image5.html']
+ >>> response.css('base').attrib['href']
+ 'http://example.com/'
- >>> response.css('a[href*=image]::attr(href)').extract()
- [u'image1.html',
- u'image2.html',
- u'image3.html',
- u'image4.html',
- u'image5.html']
+ >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+ ['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
- >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+ >>> response.css('a[href*=image]::attr(href)').getall()
+ ['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
- >>> response.css('a[href*=image] img::attr(src)').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+ >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+ ['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
+
+ >>> response.css('a[href*=image] img::attr(src)').getall()
+ ['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
+
+.. _topics-selectors-css-extensions:
+
+Extensions to CSS Selectors
+---------------------------
+
+Per W3C standards, `CSS selectors`_ do not support selecting text nodes
+or attribute values.
+But selecting these is so essential in a web scraping context
+that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
+
+* to select text nodes, use ``::text``
+* to select attribute values, use ``::attr(name)`` where *name* is the
+ name of the attribute that you want the value of
+
+.. warning::
+ These pseudo-elements are Scrapy-/Parsel-specific.
+ They will most probably not work with other libraries like
+ `lxml`_ or `PyQuery`_.
+
+.. _PyQuery: https://pypi.python.org/pypi/pyquery
+
+Examples:
+
+* ``title::text`` selects children text nodes of a descendant ```` element::
+
+ >>> response.css('title::text').get()
+ 'Example website'
+
+* ``*::text`` selects all descendant text nodes of the current selector context::
+
+ >>> response.css('#images *::text').getall()
+ ['\n ',
+ 'Name: My image 1 ',
+ '\n ',
+ 'Name: My image 2 ',
+ '\n ',
+ 'Name: My image 3 ',
+ '\n ',
+ 'Name: My image 4 ',
+ '\n ',
+ 'Name: My image 5 ',
+ '\n ']
+
+* ``foo::text`` returns no results if ``foo`` element exists, but contains
+ no text (i.e. text is empty)::
+
+ >>> response.css('img::text').getall()
+ []
+
+ This means ``.css('foo::text').get()`` could return None even if an element
+ exists. Use ``default=''`` if you always want a string::
+
+ >>> response.css('img::text').get()
+ >>> response.css('img::text').get(default='')
+ ''
+
+* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+
+ >>> response.css('a::attr(href)').getall()
+ ['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
+
+.. note::
+ See also: :ref:`selecting-attributes`.
+
+.. note::
+ You cannot chain these pseudo-elements. But in practice it would not
+ make much sense: text nodes do not have attributes, and attribute values
+ are string values already and do not have children nodes.
+
+.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
.. _topics-selectors-nesting-selectors:
@@ -206,22 +321,65 @@ of the same type, so you can call the selection methods for those selectors
too. Here's an example::
>>> links = response.xpath('//a[contains(@href, "image")]')
- >>> links.extract()
- [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
- u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
- u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
- u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
- u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+ >>> links.getall()
+ ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+ '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+ '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+ '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+ '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
- ... args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract())
- ... print 'Link number %d points to url %s and image %s' % args
+ ... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+ ... print('Link number %d points to url %r and image %r' % args)
- Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
- Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg']
- Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg']
- Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg']
- Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg']
+ Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+ Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+ Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+ Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+ Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+
+.. _selecting-attributes:
+
+Selecting element attributes
+----------------------------
+
+There are several ways to get a value of an attribute. First, one can use
+XPath syntax::
+
+ >>> response.xpath("//a/@href").getall()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+XPath syntax has a few advantages: it is a standard XPath feature, and
+``@attributes`` can be used in other parts of an XPath expression - e.g.
+it is possible to filter by attribute value.
+
+Scrapy also provides an extension to CSS selectors (``::attr(...)``)
+which allows to get attribute values::
+
+ >>> response.css('a::attr(href)').getall()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+In addition to that, there is a ``.attrib`` property of Selector.
+You can use it if you prefer to lookup attributes in Python
+code, without using XPaths or CSS extensions::
+
+ >>> [a.attrib['href'] for a in response.css('a')]
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+This property is also available on SelectorList; it returns a dictionary
+with attributes of a first matching element. It is convenient to use when
+a selector is expected to give a single result (e.g. when selecting by element
+ID, or when selecting an unique element on a page)::
+
+ >>> response.css('base').attrib
+ {'href': 'http://example.com/'}
+ >>> response.css('base').attrib['href']
+ 'http://example.com/'
+
+``.attrib`` property of an empty SelectorList is empty::
+
+ >>> response.css('foo').attrib
+ {}
Using selectors with regular expressions
----------------------------------------
@@ -235,17 +393,83 @@ Here's an example used to extract image names from the :ref:`HTML code
<topics-selectors-htmlcode>` above::
>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
- [u'My image 1',
- u'My image 2',
- u'My image 3',
- u'My image 4',
- u'My image 5']
+ ['My image 1',
+ 'My image 2',
+ 'My image 3',
+ 'My image 4',
+ 'My image 5']
-There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
-named ``.re_first()``. Use it to extract just the first matching string::
+There's an additional helper reciprocating ``.get()`` (and its
+alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
+Use it to extract just the first matching string::
>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
- u'My image 1'
+ 'My image 1'
+
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts
+and tutorials are using them as well. These methods are still supported
+by Scrapy, there are **no plans** to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+ >>> response.css('a::attr(href)').get()
+ 'image1.html'
+ >>> response.css('a::attr(href)').extract_first()
+ 'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+ >>> response.css('a::attr(href)').getall()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+ >>> response.css('a::attr(href)').extract()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+ >>> response.css('a::attr(href)')[0].get()
+ 'image1.html'
+ >>> response.css('a::attr(href)')[0].extract()
+ 'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+ >>> response.css('a::attr(href)')[0].getall()
+ ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
+
+.. _topics-selectors-xpaths:
+
+Working with XPaths
+===================
+
+Here are some tips which may help you to use XPath with Scrapy selectors
+effectively. If you are not much familiar with XPath yet,
+you may want to take a look first at this `XPath tutorial`_.
+
+.. note::
+ Some of the tips are based on `this post from ScrapingHub's blog`_.
+
+.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+
.. _topics-selectors-relative-xpaths:
@@ -266,23 +490,131 @@ it actually extracts all ``<p>`` elements from the document, not only those
inside ``<div>`` elements::
>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
- ... print p.extract()
+ ... print(p.get())
This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
>>> for p in divs.xpath('.//p'): # extracts all <p> inside
- ... print p.extract()
+ ... print(p.get())
Another common case would be to extract all direct ``<p>`` children::
>>> for p in divs.xpath('p'):
- ... print p.extract()
+ ... print(p.get())
For more details about relative XPaths see the `Location Paths`_ section in the
XPath specification.
.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
+When querying by class, consider using CSS
+------------------------------------------
+
+Because an element can contain multiple CSS classes, the XPath way to select elements
+by class is the rather verbose::
+
+ *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
+
+If you use ``@class='someclass'`` you may end up missing elements that have
+other classes, and if you just use ``contains(@class, 'someclass')`` to make up
+for that you may end up with more elements that you want, if they have a different
+class name that shares the string ``someclass``.
+
+As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
+you can just select by class using CSS and then switch to XPath when needed::
+
+ >>> from scrapy import Selector
+ >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+ >>> sel.css('.shout').xpath('./time/@datetime').getall()
+ ['2014-07-23 19:00']
+
+This is cleaner than using the verbose XPath trick shown above. Just remember
+to use the ``.`` in the XPath expressions that will follow.
+
+Beware of the difference between //node[1] and (//node)[1]
+----------------------------------------------------------
+
+``//node[1]`` selects all the nodes occurring first under their respective parents.
+
+``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
+
+Example::
+
+ >>> from scrapy import Selector
+ >>> sel = Selector(text="""
+ ....: <ul class="list">
+ ....: <li>1</li>
+ ....: <li>2</li>
+ ....: <li>3</li>
+ ....: </ul>
+ ....: <ul class="list">
+ ....: <li>4</li>
+ ....: <li>5</li>
+ ....: <li>6</li>
+ ....: </ul>""")
+ >>> xp = lambda x: sel.xpath(x).getall()
+
+This gets all first ``<li>`` elements under whatever it is its parent::
+
+ >>> xp("//li[1]")
+ ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>`` element in the whole document::
+
+ >>> xp("(//li)[1]")
+ ['<li>1</li>']
+
+This gets all first ``<li>`` elements under an ``<ul>`` parent::
+
+ >>> xp("//ul/li[1]")
+ ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document::
+
+ >>> xp("(//ul/li)[1]")
+ ['<li>1</li>']
+
+Using text nodes in a condition
+-------------------------------
+
+When you need to use the text content as argument to an `XPath string function`_,
+avoid using ``.//text()`` and use just ``.`` instead.
+
+This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
+And when a node-set is converted to a string, which happens when it is passed as argument to
+a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+
+Example::
+
+ >>> from scrapy import Selector
+ >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+
+Converting a *node-set* to string::
+
+ >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+ ['Click here to go to the ', 'Next Page']
+ >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+ ['Click here to go to the ']
+
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+
+ >>> sel.xpath("//a[1]").getall() # select the first node
+ ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+ >>> sel.xpath("string(//a[1])").getall() # convert it to string
+ ['Click here to go to the Next Page']
+
+So, using the ``.//text()`` node-set won't select anything in this case::
+
+ >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+ []
+
+But using the ``.`` to mean the node, works::
+
+ >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+ ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+
.. _topics-selectors-xpath-variables:
Variables in XPath expressions
@@ -298,14 +630,14 @@ Here's an example to match an element based on its "id" attribute value,
without hard-coding it (that was shown previously)::
>>> # `$val` used in the expression, a `val` argument needs to be passed
- >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
- u'Name: My image 1 '
+ >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+ 'Name: My image 1 '
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
five ``<a>`` children (here we pass the value ``5`` as an integer)::
- >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
- u'images'
+ >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+ 'images'
All variable references must have a binding value when calling ``.xpath()``
(otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -314,13 +646,78 @@ This is done by passing as many named arguments as necessary.
`parsel`_, the library powering Scrapy selectors, has more details and examples
on `XPath variables`_.
-.. _parsel: https://parsel.readthedocs.io/
.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
+
+.. _removing-namespaces:
+
+Removing namespaces
+-------------------
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names, to write more
+simple/convenient XPaths. You can use the
+:meth:`Selector.remove_namespaces` method for that.
+
+Let's show an example that illustrates this with the Python Insider blog atom feed.
+
+.. highlight:: sh
+
+First, we open the shell with the url we want to scrape::
+
+ $ scrapy shell https://feeds.feedburner.com/PythonInsider
+
+This is how the file starts::
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <?xml-stylesheet ...
+ <feed xmlns="http://www.w3.org/2005/Atom"
+ xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/"
+ xmlns:blogger="http://schemas.google.com/blogger/2008"
+ xmlns:georss="http://www.georss.org/georss"
+ xmlns:gd="http://schemas.google.com/g/2005"
+ xmlns:thr="http://purl.org/syndication/thread/1.0"
+ xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
+ ...
+
+You can see several namespace declarations including a default
+"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for
+"http://schemas.google.com/g/2005".
+
+.. highlight:: python
+
+Once in the shell we can try selecting all ``<link>`` objects and see that it
+doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+
+ >>> response.xpath("//link")
+ []
+
+But once we call the :meth:`Selector.remove_namespaces` method, all
+nodes can be accessed directly by their names::
+
+ >>> response.selector.remove_namespaces()
+ >>> response.xpath("//link")
+ [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+ <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
+ ...
+
+If you wonder why the namespace removal procedure isn't always called by default
+instead of having to call it manually, this is because of two reasons, which, in order
+of relevance, are:
+
+1. Removing namespaces requires to iterate and modify all nodes in the
+ document, which is a reasonably expensive operation to perform by default
+ for all documents crawled by Scrapy
+
+2. There could be some cases where using namespaces is actually required, in
+ case some element names clash between namespaces. These cases are very rare
+ though.
+
+
Using EXSLT extensions
----------------------
-Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions
and come with these pre-registered namespaces to use in XPath expressions:
@@ -340,7 +737,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit::
>>> from scrapy import Selector
- >>> doc = """
+ >>> doc = u"""
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
@@ -352,10 +749,10 @@ Example selecting links in list item with a "class" attribute ending with a digi
... </div>
... """
>>> sel = Selector(text=doc, type="html")
- >>> sel.xpath('//li//@href').extract()
- [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
- >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
- [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+ >>> sel.xpath('//li//@href').getall()
+ ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+ >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+ ['link1.html', 'link2.html', 'link4.html', 'link5.html']
>>>
.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
@@ -372,7 +769,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from http://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
- >>> doc = """
+ >>> doc = u"""
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@@ -424,33 +821,33 @@ with groups of itemscopes and corresponding itemprops::
... """
>>> sel = Selector(text=doc, type="html")
>>> for scope in sel.xpath('//div[@itemscope]'):
- ... print "current scope:", scope.xpath('@itemtype').extract()
+ ... print("current scope:", scope.xpath('@itemtype').getall())
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
- ... print " properties:", props.extract()
- ... print
+ ... print(" properties: %s" % (props.getall()))
+ ... print("")
- current scope: [u'http://schema.org/Product']
- properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+ current scope: ['http://schema.org/Product']
+ properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
- current scope: [u'http://schema.org/AggregateRating']
- properties: [u'ratingValue', u'reviewCount']
+ current scope: ['http://schema.org/AggregateRating']
+ properties: ['ratingValue', 'reviewCount']
- current scope: [u'http://schema.org/Offer']
- properties: [u'price', u'availability']
+ current scope: ['http://schema.org/Offer']
+ properties: ['price', 'availability']
- current scope: [u'http://schema.org/Review']
- properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+ current scope: ['http://schema.org/Review']
+ properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
- current scope: [u'http://schema.org/Rating']
- properties: [u'worstRating', u'ratingValue', u'bestRating']
+ current scope: ['http://schema.org/Rating']
+ properties: ['worstRating', 'ratingValue', 'bestRating']
- current scope: [u'http://schema.org/Review']
- properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+ current scope: ['http://schema.org/Review']
+ properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
- current scope: [u'http://schema.org/Rating']
- properties: [u'worstRating', u'ratingValue', u'bestRating']
+ current scope: ['http://schema.org/Rating']
+ properties: ['worstRating', 'ratingValue', 'bestRating']
>>>
@@ -462,127 +859,44 @@ inside another ``itemscope``.
.. _regular expressions: http://exslt.org/regexp/index.html
.. _set manipulation: http://exslt.org/set/index.html
+Other XPath extensions
+----------------------
-Some XPath tips
----------------
+Scrapy selectors also provide a sorely missed XPath extension function
+``has-class`` that returns ``True`` for nodes that have all of the specified
+HTML classes.
-Here are some tips that you may find useful when using XPath
-with Scrapy selectors, based on `this post from ScrapingHub's blog`_.
-If you are not much familiar with XPath yet,
-you may want to take a look first at this `XPath tutorial`_.
+.. highlight:: html
+For the following HTML::
-.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+ <p class="foo bar-baz">First</p>
+ <p class="foo">Second</p>
+ <p class="bar">Third</p>
+ <p>Fourth</p>
+.. highlight:: python
-Using text nodes in a condition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can use it like this::
-When you need to use the text content as argument to an `XPath string function`_,
-avoid using ``.//text()`` and use just ``.`` instead.
-
-This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
-And when a node-set is converted to a string, which happens when it is passed as argument to
-a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
-
-Example::
-
- >>> from scrapy import Selector
- >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
-
-Converting a *node-set* to string::
-
- >>> sel.xpath('//a//text()').extract() # take a peek at the node-set
- [u'Click here to go to the ', u'Next Page']
- >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string
- [u'Click here to go to the ']
-
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
-
- >>> sel.xpath("//a[1]").extract() # select the first node
- [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
- >>> sel.xpath("string(//a[1])").extract() # convert it to string
- [u'Click here to go to the Next Page']
-
-So, using the ``.//text()`` node-set won't select anything in this case::
-
- >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract()
+ >>> response.xpath('//p[has-class("foo")]')
+ [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+ <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+ >>> response.xpath('//p[has-class("foo", "bar-baz")]')
+ [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+ >>> response.xpath('//p[has-class("foo", "bar")]')
[]
-But using the ``.`` to mean the node, works::
+So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
+``p.foo.bar-baz``. Please note, that it is slower in most of the cases,
+because it's a pure-Python function that's invoked for every node in question
+whereas the CSS lookup is translated into XPath and thus runs more efficiently,
+so performance-wise its uses are limited to situations that are not easily
+described with CSS selectors.
- >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
- [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+Parsel also simplifies adding your own XPath extensions.
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
-
-Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``//node[1]`` selects all the nodes occurring first under their respective parents.
-
-``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
-
-Example::
-
- >>> from scrapy import Selector
- >>> sel = Selector(text="""
- ....: <ul class="list">
- ....: <li>1</li>
- ....: <li>2</li>
- ....: <li>3</li>
- ....: </ul>
- ....: <ul class="list">
- ....: <li>4</li>
- ....: <li>5</li>
- ....: <li>6</li>
- ....: </ul>""")
- >>> xp = lambda x: sel.xpath(x).extract()
-
-This gets all first ``<li>`` elements under whatever it is its parent::
-
- >>> xp("//li[1]")
- [u'<li>1</li>', u'<li>4</li>']
-
-And this gets the first ``<li>`` element in the whole document::
-
- >>> xp("(//li)[1]")
- [u'<li>1</li>']
-
-This gets all first ``<li>`` elements under an ``<ul>`` parent::
-
- >>> xp("//ul/li[1]")
- [u'<li>1</li>', u'<li>4</li>']
-
-And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document::
-
- >>> xp("(//ul/li)[1]")
- [u'<li>1</li>']
-
-When querying by class, consider using CSS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Because an element can contain multiple CSS classes, the XPath way to select elements
-by class is the rather verbose::
-
- *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
-
-If you use ``@class='someclass'`` you may end up missing elements that have
-other classes, and if you just use ``contains(@class, 'someclass')`` to make up
-for that you may end up with more elements that you want, if they have a different
-class name that shares the string ``someclass``.
-
-As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
-
- >>> from scrapy import Selector
- >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
- >>> sel.css('.shout').xpath('./time/@datetime').extract()
- [u'2014-07-23 19:00']
-
-This is cleaner than using the verbose XPath trick shown above. Just remember
-to use the ``.`` in the XPath expressions that will follow.
+.. autofunction:: parsel.xpathfuncs.set_xpathfunc
.. _topics-selectors-ref:
@@ -596,132 +910,79 @@ Built-in Selectors reference
Selector objects
----------------
-.. class:: Selector(response=None, text=None, type=None)
+.. autoclass:: Selector
- An instance of :class:`Selector` is a wrapper over response to select
- certain parts of its content.
-
- ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
- :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
- extracting data.
-
- ``text`` is a unicode string or utf-8 encoded text for cases when a
- ``response`` isn't available. Using ``text`` and ``response`` together is
- undefined behavior.
-
- ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
-
- If ``type`` is ``None``, the selector automatically chooses the best type
- based on ``response`` type (see below), or defaults to ``"html"`` in case it
- is used together with ``text``.
-
- If ``type`` is ``None`` and a ``response`` is passed, the selector type is
- inferred from the response type as follows:
-
- * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
- * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
- * ``"html"`` for anything else
-
- Otherwise, if ``type`` is set, the selector type will be forced and no
- detection will occur.
-
- .. method:: xpath(query)
-
- Find nodes matching the xpath ``query`` and return the result as a
- :class:`SelectorList` instance with all elements flattened. List
- elements implement :class:`Selector` interface too.
-
- ``query`` is a string containing the XPATH query to apply.
+ .. automethod:: xpath
.. note::
For convenience, this method can be called as ``response.xpath()``
- .. method:: css(query)
-
- Apply the given CSS selector and return a :class:`SelectorList` instance.
-
- ``query`` is a string containing the CSS selector to apply.
-
- In the background, CSS queries are translated into XPath queries using
- `cssselect`_ library and run ``.xpath()`` method.
+ .. automethod:: css
.. note::
- For convenience this method can be called as ``response.css()``
+ For convenience, this method can be called as ``response.css()``
- .. method:: extract()
+ .. automethod:: get
- Serialize and return the matched nodes as a list of unicode strings.
- Percent encoded content is unquoted.
+ See also: :ref:`old-extraction-api`
- .. method:: re(regex)
+ .. autoattribute:: attrib
- Apply the given regex and return a list of unicode strings with the
- matches.
+ See also: :ref:`selecting-attributes`.
- ``regex`` can be either a compiled regular expression or a string which
- will be compiled to a regular expression using ``re.compile(regex)``
+ .. automethod:: re
- .. note::
+ .. automethod:: re_first
- Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``).
+ .. automethod:: register_namespace
- .. method:: register_namespace(prefix, uri)
+ .. automethod:: remove_namespaces
- Register the given namespace to be used in this :class:`Selector`.
- Without registering namespaces you can't select or extract data from
- non-standard namespaces. See examples below.
+ .. automethod:: __bool__
- .. method:: remove_namespaces()
-
- Remove all namespaces, allowing to traverse the document using
- namespace-less xpaths. See example below.
-
- .. method:: __nonzero__()
-
- Returns ``True`` if there is any real content selected or ``False``
- otherwise. In other words, the boolean value of a :class:`Selector` is
- given by the contents it selects.
+ .. automethod:: getall
+ This method is added to Selector for consistency; it is more useful
+ with SelectorList. See also: :ref:`old-extraction-api`
SelectorList objects
--------------------
-.. class:: SelectorList
+.. autoclass:: SelectorList
- The :class:`SelectorList` class is a subclass of the builtin ``list``
- class, which provides a few additional methods.
+ .. automethod:: xpath
- .. method:: xpath(query)
+ .. automethod:: css
- Call the ``.xpath()`` method for each element in this list and return
- their results flattened as another :class:`SelectorList`.
+ .. automethod:: getall
- ``query`` is the same argument as the one in :meth:`Selector.xpath`
+ See also: :ref:`old-extraction-api`
- .. method:: css(query)
+ .. automethod:: get
- Call the ``.css()`` method for each element in this list and return
- their results flattened as another :class:`SelectorList`.
+ See also: :ref:`old-extraction-api`
- ``query`` is the same argument as the one in :meth:`Selector.css`
+ .. automethod:: re
- .. method:: extract()
+ .. automethod:: re_first
- Call the ``.extract()`` method for each element in this list and return
- their results flattened, as a list of unicode strings.
+ .. autoattribute:: attrib
- .. method:: re()
+ See also: :ref:`selecting-attributes`.
- Call the ``.re()`` method for each element in this list and return
- their results flattened, as a list of unicode strings.
+.. _selector-examples:
+Examples
+========
+
+.. _selector-examples-html:
Selector examples on HTML response
----------------------------------
-Here's a couple of :class:`Selector` examples to illustrate several concepts.
+Here are some :class:`Selector` examples to illustrate several concepts.
In all cases, we assume there is already a :class:`Selector` instantiated with
a :class:`~scrapy.http.HtmlResponse` object like this::
@@ -735,20 +996,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
- sel.xpath("//h1").extract() # this includes the h1 tag
- sel.xpath("//h1/text()").extract() # this excludes the h1 tag
+ sel.xpath("//h1").getall() # this includes the h1 tag
+ sel.xpath("//h1/text()").getall() # this excludes the h1 tag
3. Iterate over all ``<p>`` tags and print their class attribute::
for node in sel.xpath("//p"):
- print node.xpath("@class").extract()
+ print(node.attrib['class'])
+
+
+.. _selector-examples-xml:
Selector examples on XML response
---------------------------------
-Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already a :class:`Selector` instantiated with an
-:class:`~scrapy.http.XmlResponse` object like this::
+Here are some examples to illustrate concepts for :class:`Selector` objects
+instantiated with an :class:`~scrapy.http.XmlResponse` object::
sel = Selector(xml_response)
@@ -761,53 +1024,6 @@ assume there is already a :class:`Selector` instantiated with an
a namespace::
sel.register_namespace("g", "http://base.google.com/ns/1.0")
- sel.xpath("//g:price").extract()
-
-.. _removing-namespaces:
-
-Removing namespaces
--------------------
-
-When dealing with scraping projects, it is often quite convenient to get rid of
-namespaces altogether and just work with element names, to write more
-simple/convenient XPaths. You can use the
-:meth:`Selector.remove_namespaces` method for that.
-
-Let's show an example that illustrates this with GitHub blog atom feed.
-
-.. highlight:: sh
-
-First, we open the shell with the url we want to scrape::
-
- $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: python
-
-Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
-
- >>> response.xpath("//link")
- []
-
-But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
-
- >>> response.selector.remove_namespaces()
- >>> response.xpath("//link")
- [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
- <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
- ...
-
-If you wonder why the namespace removal procedure isn't always called by default
-instead of having to call it manually, this is because of two reasons, which, in order
-of relevance, are:
-
-1. Removing namespaces requires to iterate and modify all nodes in the
- document, which is a reasonably expensive operation to perform for all
- documents crawled by Scrapy
-
-2. There could be some cases where using namespaces is actually required, in
- case some element names clash between namespaces. These cases are very rare
- though.
+ sel.xpath("//g:price").getall()
.. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1f1217770..47b6cf13d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -871,7 +871,7 @@ LOG_STDOUT
Default: ``False``
If ``True``, all standard output (and error) of your process will be redirected
-to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
+to the log. For example if you ``print('hello')`` it will appear in the Scrapy
log.
.. setting:: LOG_SHORT_NAMES
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 11ab199f2..68a0b19b5 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -179,13 +179,13 @@ all start with the ``[s]`` prefix)::
After that, we can start playing with the objects::
- >>> response.xpath('//title/text()').extract_first()
+ >>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> fetch("https://reddit.com")
- >>> response.xpath('//title/text()').extract()
- ['reddit: the front page of the internet']
+ >>> response.xpath('//title/text()').get()
+ 'reddit: the front page of the internet'
>>> request = request.replace(method="POST")
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 697732b47..a08dc30f2 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::
]
def parse(self, response):
- for h3 in response.xpath('//h3').extract():
+ for h3 in response.xpath('//h3').getall():
yield {"title": h3}
- for url in response.xpath('//a/@href').extract():
- yield scrapy.Request(url, callback=self.parse)
+ for href in response.xpath('//a/@href').getall():
+ yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
@@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::
yield scrapy.Request('http://www.example.com/3.html', self.parse)
def parse(self, response):
- for h3 in response.xpath('//h3').extract():
+ for h3 in response.xpath('//h3').getall():
yield MyItem(title=h3)
- for url in response.xpath('//a/@href').extract():
- yield scrapy.Request(url, callback=self.parse)
+ for href in response.xpath('//a/@href').getall():
+ yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules::
self.logger.info('Hi, this is an item page! %s', response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
- item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
- item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
+ item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
+ item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
return item
@@ -545,12 +545,12 @@ These spiders are pretty easy to use, let's have a look at one example::
itertag = 'item'
def parse_node(self, response, node):
- self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
+ self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
item = TestItem()
- item['id'] = node.xpath('@id').extract()
- item['name'] = node.xpath('name').extract()
- item['description'] = node.xpath('description').extract()
+ item['id'] = node.xpath('@id').get()
+ item['name'] = node.xpath('name').get()
+ item['description'] = node.xpath('description').get()
return item
Basically what we did up there was to create a spider that downloads a feed from
diff --git a/requirements-py2.txt b/requirements-py2.txt
index 03b33d02d..0771aae3a 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -6,5 +6,5 @@ queuelib
w3lib>=1.17.0
six>=1.5.2
PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index b38c4cc09..5a5d4c95a 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -6,5 +6,5 @@ queuelib>=1.1.1
w3lib>=1.17.0
six>=1.5.2
PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
service_identity
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index f4ca4262a..5fa6b771c 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
base_url = get_base_url(response)
body = u''.join(f
for x in self.restrict_xpaths
- for f in response.xpath(x).extract()
+ for f in response.xpath(x).getall()
).encode(response.encoding, errors='xmlcharrefreplace')
else:
body = response.body
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index e73413318..a7c75a46a 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -181,7 +181,7 @@ class ItemLoader(object):
def _get_xpathvalues(self, xpaths, **kw):
self._check_selector_method()
xpaths = arg_to_iter(xpaths)
- return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
+ return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
def add_css(self, field_name, css, *processors, **kw):
values = self._get_cssvalues(css, **kw)
@@ -198,6 +198,6 @@ class ItemLoader(object):
def _get_cssvalues(self, csss, **kw):
self._check_selector_method()
csss = arg_to_iter(csss)
- return flatten(self.selector.css(css).extract() for css in csss)
+ return flatten(self.selector.css(css).getall() for css in csss)
XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 64cb0232c..8f6cb1d79 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -27,6 +27,10 @@ def _response_from_text(text, st):
class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
+ """
+ The :class:`SelectorList` class is a subclass of the builtin ``list``
+ class, which provides a few additional methods.
+ """
@deprecated(use_instead='.extract()')
def extract_unquoted(self):
return [x.extract_unquoted() for x in self]
@@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
class Selector(_ParselSelector, object_ref):
+ """
+ An instance of :class:`Selector` is a wrapper over response to select
+ certain parts of its content.
+
+ ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
+ :class:`~scrapy.http.XmlResponse` object that will be used for selecting
+ and extracting data.
+
+ ``text`` is a unicode string or utf-8 encoded text for cases when a
+ ``response`` isn't available. Using ``text`` and ``response`` together is
+ undefined behavior.
+
+ ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+ or ``None`` (default).
+
+ If ``type`` is ``None``, the selector automatically chooses the best type
+ based on ``response`` type (see below), or defaults to ``"html"`` in case it
+ is used together with ``text``.
+
+ If ``type`` is ``None`` and a ``response`` is passed, the selector type is
+ inferred from the response type as follows:
+
+ * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
+ * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+ * ``"html"`` for anything else
+
+ Otherwise, if ``type`` is set, the selector type will be forced and no
+ detection will occur.
+ """
__slots__ = ['response']
selectorlist_cls = SelectorList
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 802cb88a1..878425125 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -14,8 +14,8 @@ class $classname(CrawlSpider):
)
def parse_item(self, response):
- i = {}
- #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
- #i['name'] = response.xpath('//div[@id="name"]').extract()
- #i['description'] = response.xpath('//div[@id="description"]').extract()
- return i
+ item = {}
+ #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+ #item['name'] = response.xpath('//div[@id="name"]').get()
+ #item['description'] = response.xpath('//div[@id="description"]').get()
+ return item
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 7c2ff8850..863c9772f 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
itertag = 'item' # change it accordingly
def parse_node(self, response, selector):
- i = {}
- #i['url'] = selector.select('url').extract()
- #i['name'] = selector.select('name').extract()
- #i['description'] = selector.select('description').extract()
- return i
+ item = {}
+ #item['url'] = selector.select('url').get()
+ #item['name'] = selector.select('name').get()
+ #item['description'] = selector.select('description').get()
+ return item
diff --git a/setup.py b/setup.py
index c37919cda..8c47f67ce 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
'pyOpenSSL',
'cssselect>=0.9',
'six>=1.5.2',
- 'parsel>=1.4',
+ 'parsel>=1.5',
'PyDispatcher>=2.0.5',
'service_identity',
],
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 3e27d6abd..36baacfbd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_response_selector_html(self):
- xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
+ xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
self.assertEqual(out.strip(), b'Works')
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 820758dc9..3b90e3dac 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
self.assertIs(response.selector.response, response)
self.assertEqual(
- response.selector.xpath("//title/text()").extract(),
+ response.selector.xpath("//title/text()").getall(),
[u'Some page']
)
self.assertEqual(
- response.selector.css("title::text").extract(),
+ response.selector.css("title::text").getall(),
[u'Some page']
)
self.assertEqual(
@@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
- response.xpath("//title/text()").extract(),
- response.selector.xpath("//title/text()").extract(),
+ response.xpath("//title/text()").getall(),
+ response.selector.xpath("//title/text()").getall(),
)
self.assertEqual(
- response.css("title::text").extract(),
- response.selector.css("title::text").extract(),
+ response.css("title::text").getall(),
+ response.selector.css("title::text").getall(),
)
def test_selector_shortcuts_kwargs(self):
@@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
- response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
- response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
+ response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
+ response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
)
self.assertEqual(
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
- pclass="content", pcount=1).extract(),
- response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
+ pclass="content", pcount=1).getall(),
+ response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
)
def test_urljoin_with_base_url(self):
@@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
self.assertIs(response.selector.response, response)
self.assertEqual(
- response.selector.xpath("//elem/text()").extract(),
+ response.selector.xpath("//elem/text()").getall(),
[u'value']
)
@@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
- response.xpath("//elem/text()").extract(),
- response.selector.xpath("//elem/text()").extract(),
+ response.xpath("//elem/text()").getall(),
+ response.selector.xpath("//elem/text()").getall(),
)
def test_selector_shortcuts_kwargs(self):
@@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
- response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
- response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+ response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
+ response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
)
response.selector.register_namespace('s2', 'http://scrapy.org')
self.assertEqual(
- response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
- response.selector.xpath("//s2:elem/text()").extract(),
+ response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
+ response.selector.xpath("//s2:elem/text()").getall(),
)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 3b5714058..8b58e4dbd 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
nl = l.nested_xpath("//header")
nl.add_xpath('name', 'div/text()')
nl.add_css('name_div', '#id')
- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+ nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
@@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
nl = l.nested_css("header")
nl.add_xpath('name', 'div/text()')
nl.add_css('name_div', '#id')
- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+ nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5985a6f3e..fb72c9d6d 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
for href in response.xpath('''
//table[thead/tr/th="Filename"]
/tbody//a/@href
- ''').extract()],
+ ''').getall()],
}
yield item
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 526660cc8..bc4baf7ea 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
for x in xl:
assert isinstance(x, Selector)
- self.assertEqual(sel.xpath('//input').extract(),
- [x.extract() for x in sel.xpath('//input')])
+ self.assertEqual(sel.xpath('//input').getall(),
+ [x.get() for x in sel.xpath('//input')])
- self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
+ self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
[u'a'])
- self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+ self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
[u'12.0'])
- self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
+ self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
[u'xpathrules'])
- self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+ self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
[u'12'])
def test_root_base_url(self):
@@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
text = b'<div><img src="a.jpg"><p>Hello</div>'
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'xml')
- self.assertEqual(sel.xpath("//div").extract(),
+ self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'html')
- self.assertEqual(sel.xpath("//div").extract(),
+ self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
def test_http_header_encoding_precedence(self):
@@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
x = Selector(response)
- self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
[u'\xa3'])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
- r1 = TextResponse('http://www.example.com', \
- body=b'<html><p>an Jos\xe9 de</p><html>', \
+ r1 = TextResponse('http://www.example.com',
+ body=b'<html><p>an Jos\xe9 de</p><html>',
encoding='utf-8')
- Selector(r1).xpath('//text()').extract()
+ Selector(r1).xpath('//text()').getall()
def test_weakref_slots(self):
"""Check that classes are using slots and are weak-referenceable"""
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 929e0fea8..f26da2334 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
def parse_node(self, response, selector):
yield {
- 'loc': selector.xpath('a:loc/text()').extract(),
- 'updated': selector.xpath('b:updated/text()').extract(),
- 'other': selector.xpath('other/@value').extract(),
- 'custom': selector.xpath('other/@b:custom').extract(),
+ 'loc': selector.xpath('a:loc/text()').getall(),
+ 'updated': selector.xpath('b:updated/text()').getall(),
+ 'other': selector.xpath('other/@value').getall(),
+ 'custom': selector.xpath('other/@b:custom').getall(),
}
for iterator in ('iternodes', 'xml'):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f953076b8..2d845697e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse(url="http://example.com", body=body)
attrs = []
for x in self.xmliter(response, 'product'):
- attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
+ attrs.append((
+ x.attrib['id'],
+ x.xpath("name/text()").getall(),
+ x.xpath("./type/text()").getall()))
self.assertEqual(attrs,
- [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
+ [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
def test_xmliter_unusual_node(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
</root>
"""
response = XmlResponse(url="http://example.com", body=body)
- nodenames = [e.xpath('name()').extract()
+ nodenames = [e.xpath('name()').getall()
for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
@@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
attrs = []
for x in self.xmliter(r, u'þingflokkur'):
- attrs.append((x.xpath('@id').extract(),
- x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
- x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+ attrs.append((x.attrib['id'],
+ x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
+ x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
self.assertEqual(attrs,
- [([u'26'], [u'-'], [u'80']),
- ([u'21'], [u'Ab'], [u'76']),
- ([u'27'], [u'A'], [u'27'])])
+ [(u'26', [u'-'], [u'80']),
+ (u'21', [u'Ab'], [u'76']),
+ (u'27', [u'A'], [u'27'])])
def test_xmliter_text(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
- self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
+ self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
def test_xmliter_namespaces(self):
@@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
node = next(my_iter)
node.register_namespace('g', 'http://base.google.com/ns/1.0')
- self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
- self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
- self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
- self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
- self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
- self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
- self.assertEqual(node.xpath('image_link/text()').extract(), [])
- self.assertEqual(node.xpath('id/text()').extract(), [])
- self.assertEqual(node.xpath('price/text()').extract(), [])
+ self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
+ self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
+ self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+ self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+ self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
+ self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
+ self.assertEqual(node.xpath('image_link/text()').getall(), [])
+ self.assertEqual(node.xpath('id/text()').getall(), [])
+ self.assertEqual(node.xpath('price/text()').getall(), [])
def test_xmliter_exception(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
response = XmlResponse('http://www.example.com', body=body)
self.assertEqual(
- next(self.xmliter(response, 'item')).extract(),
+ next(self.xmliter(response, 'item')).get(),
u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
)
@@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
node = next(namespace_iter)
- self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+ self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
node = next(namespace_iter)
- self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
+ self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
def test_xmliter_namespaces_prefix(self):
body = b"""\
@@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
node = next(my_iter)
- self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
- self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
- self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
+ self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
+ self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
+ self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
node = next(my_iter)
- self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+ self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, 'product')