From 25ac4691b414e7c18a4e9dec3bb6a85563d2488d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 9 Aug 2018 03:32:46 +0500
Subject: [PATCH 01/18] require parsel 1.5+

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 setup.py             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 03b33d02d..0771aae3a 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -6,5 +6,5 @@ queuelib
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index b38c4cc09..5a5d4c95a 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -6,5 +6,5 @@ queuelib>=1.1.1
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/setup.py b/setup.py
index c37919cda..8c47f67ce 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=1.4',
+        'parsel>=1.5',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 0ccead6681c9e2bf1902cbc1d4bde543be7d73e7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:36 +0500
Subject: [PATCH 02/18] DOC more Python 3 in examples

---
 docs/topics/commands.rst  |  6 ++---
 docs/topics/items.rst     |  6 ++---
 docs/topics/jobs.rst      |  4 ++--
 docs/topics/loaders.rst   | 14 ++++++------
 docs/topics/selectors.rst | 48 +++++++++++++++++++--------------------
 docs/topics/settings.rst  |  2 +-
 6 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3088017cb..ef9c45196 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -458,9 +458,9 @@ Usage example::
 
     >>> STATUS DEPTH LEVEL 1 <<<
     # Scraped Items  ------------------------------------------------------------
-    [{'name': u'Example item',
-     'category': u'Furniture',
-     'length': u'12 cm'}]
+    [{'name': 'Example item',
+     'category': 'Furniture',
+     'length': '12 cm'}]
 
     # Requests  -----------------------------------------------------------------
     []
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4423bbda2..ae44aecd3 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -86,7 +86,7 @@ Creating items
 ::
 
     >>> product = Product(name='Desktop PC', price=1000)
-    >>> print product
+    >>> print(product)
     Product(name='Desktop PC', price=1000)
 
 Getting field values
@@ -161,11 +161,11 @@ Other common tasks
 Copying items::
 
     >>> product2 = Product(product)
-    >>> print product2
+    >>> print(product2)
     Product(name='Desktop PC', price=1000)
 
     >>> product3 = product2.copy()
-    >>> print product3
+    >>> print(product3)
     Product(name='Desktop PC', price=1000)
 
 Creating dicts from items::
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 06c7fff3d..8e1574376 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -84,7 +84,7 @@ So, for example, this won't work::
         return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 But this will::
 
@@ -94,7 +94,7 @@ But this will::
 
     def other_callback(self, response):
         somearg = response.meta['somearg']
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a895b535c..f3b6aa4a1 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -678,10 +678,10 @@ Here is a list of all built-in processors:
         >>> from scrapy.loader.processors import Join
         >>> proc = Join()
         >>> proc(['one', 'two', 'three'])
-        u'one two three'
+        'one two three'
         >>> proc = Join('<br>')
         >>> proc(['one', 'two', 'three'])
-        u'one<br>two<br>three'
+        'one<br>two<br>three'
 
 .. class:: Compose(\*functions, \**default_loader_context)
 
@@ -744,9 +744,9 @@ Here is a list of all built-in processors:
         ...     return None if x == 'world' else x
         ...
         >>> from scrapy.loader.processors import MapCompose
-        >>> proc = MapCompose(filter_world, unicode.upper)
-        >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
-        [u'HELLO, u'THIS', u'IS', u'SCRAPY']
+        >>> proc = MapCompose(filter_world, str.upper)
+        >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+        ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
     constructor keyword arguments are used as default context values. See
@@ -772,7 +772,7 @@ Here is a list of all built-in processors:
         >>> import json
         >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
         >>> proc_single_json_str('{"foo": "bar"}')
-        u'bar'
+        'bar'
         >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
         >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-        [u'bar']
+        ['bar']
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ac40c3cc..25c1f0aab 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -235,17 +235,17 @@ Here's an example used to extract image names from the :ref:`HTML code
 <topics-selectors-htmlcode>` above::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
-    [u'My image 1',
-     u'My image 2',
-     u'My image 3',
-     u'My image 4',
-     u'My image 5']
+    ['My image 1',
+     'My image 2',
+     'My image 3',
+     'My image 4',
+     'My image 5']
 
 There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
 named ``.re_first()``. Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
-    u'My image 1'
+    'My image 1'
 
 .. _topics-selectors-relative-xpaths:
 
@@ -431,26 +431,26 @@ with groups of itemscopes and corresponding itemprops::
     ...     print "    properties:", props.extract()
     ...     print
 
-    current scope: [u'http://schema.org/Product']
-        properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+    current scope: ['http://schema.org/Product']
+        properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
 
-    current scope: [u'http://schema.org/AggregateRating']
-        properties: [u'ratingValue', u'reviewCount']
+    current scope: ['http://schema.org/AggregateRating']
+        properties: ['ratingValue', 'reviewCount']
 
-    current scope: [u'http://schema.org/Offer']
-        properties: [u'price', u'availability']
+    current scope: ['http://schema.org/Offer']
+        properties: ['price', 'availability']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
     >>>
 
@@ -543,22 +543,22 @@ Example::
 This gets all first ``<li>``  elements under whatever it is its parent::
 
     >>> xp("//li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element in the whole document::
 
     >>> xp("(//li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 This gets all first ``<li>``  elements under an ``<ul>``  parent::
 
     >>> xp("//ul/li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
 
     >>> xp("(//ul/li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 When querying by class, consider using CSS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1f1217770..47b6cf13d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -871,7 +871,7 @@ LOG_STDOUT
 Default: ``False``
 
 If ``True``, all standard output (and error) of your process will be redirected
-to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
+to the log. For example if you ``print('hello')`` it will appear in the Scrapy
 log.
 
 .. setting:: LOG_SHORT_NAMES

From 395d9d033a39ae9e82337ce50cffabd1ecedb702 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:57 +0500
Subject: [PATCH 03/18] add pytest temp files to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ff6e2ea65..7392ed31e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
+.pytest_cache/
 
 # Windows
 Thumbs.db

From ca27010d4f8c35d1c98259e56be797a18b044304 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:22:56 +0500
Subject: [PATCH 04/18] DOC .extract_first() -> .get()

---
 docs/intro/overview.rst |  6 ++---
 docs/intro/tutorial.rst | 49 +++++++++++++++++++++--------------------
 docs/topics/shell.rst   |  2 +-
 docs/topics/spiders.rst | 10 ++++-----
 4 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6f1c2c43f..9d7c94d39 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.xpath('span/small/text()').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.xpath('span/small/text()').get(),
                 }
 
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr("href")').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fa6dc274d..a24cf0f5b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -269,23 +269,24 @@ The other thing is that the result of calling ``.extract()`` is a list, because
 we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
 you know you just want the first result, as in this case, you can do::
 
-    >>> response.css('title::text').extract_first()
+    >>> response.css('title::text').get()
     'Quotes to Scrape'
 
 As an alternative, you could've written::
 
-    >>> response.css('title::text')[0].extract()
+    >>> response.css('title::text')[0].get()
     'Quotes to Scrape'
 
-However, using ``.extract_first()`` avoids an ``IndexError`` and returns
-``None`` when it doesn't find any element matching the selection.
+However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
+instance avoids an ``IndexError`` and returns ``None`` when it doesn't
+find any element matching the selection.
 
 There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
 Besides the :meth:`~scrapy.selector.Selector.extract` and
-:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
+:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
 expressions`::
 
@@ -314,7 +315,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
 
     >>> response.xpath('//title')
     [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
@@ -383,10 +384,10 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text::text").extract_first()
+    >>> title = quote.css("span.text::text").get()
     >>> title
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author::text").extract_first()
+    >>> author = quote.css("small.author::text").get()
     >>> author
     'Albert Einstein'
 
@@ -401,8 +402,8 @@ Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::
 
     >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text::text").extract_first()
-    ...     author = quote.css("small.author::text").extract_first()
+    ...     text = quote.css("span.text::text").get()
+    ...     author = quote.css("small.author::text").get()
     ...     tags = quote.css("div.tags a.tag::text").extract()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
@@ -434,8 +435,8 @@ in the callback, as you can see below::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
@@ -508,14 +509,14 @@ markup:
 
 We can try extracting it in the shell::
 
-    >>> response.css('li.next a').extract_first()
+    >>> response.css('li.next a').get()
     '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
 Scrapy supports a CSS extension that let's you select the attribute contents,
 like this::
 
-    >>> response.css('li.next a::attr(href)').extract_first()
+    >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
@@ -533,12 +534,12 @@ page, extracting data from it::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
@@ -584,12 +585,12 @@ As a shortcut for creating Request objects you can use
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('span small::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, callback=self.parse)
 
@@ -641,7 +642,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).extract_first().strip()
+                return response.css(query).get().strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
@@ -710,11 +711,11 @@ with a specific tag, building the URL based on the argument::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 11ab199f2..9de6abef7 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -179,7 +179,7 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
     >>> fetch("https://reddit.com")
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 697732b47..4505b7315 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules::
             self.logger.info('Hi, this is an item page! %s', response.url)
             item = scrapy.Item()
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
-            item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
-            item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
+            item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
+            item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             return item
 
 
@@ -548,9 +548,9 @@ These spiders are pretty easy to use, let's have a look at one example::
             self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
 
             item = TestItem()
-            item['id'] = node.xpath('@id').extract()
-            item['name'] = node.xpath('name').extract()
-            item['description'] = node.xpath('description').extract()
+            item['id'] = node.xpath('@id').get()
+            item['name'] = node.xpath('name').get()
+            item['description'] = node.xpath('description').get()
             return item
 
 Basically what we did up there was to create a spider that downloads a feed from

From d32c4deaa99bf0155542d1b508b5535065b5d7a9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:23:31 +0500
Subject: [PATCH 05/18] DOC update Scrapy selectors tutorial to match parsel's
 tutorial better

---
 docs/topics/selectors.rst | 471 +++++++++++++++++++++++++++-----------
 1 file changed, 340 insertions(+), 131 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 25c1f0aab..00158ecf1 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -6,7 +6,7 @@ Selectors
 
 When you're scraping web pages, the most common task you need to perform is
 to extract data from the HTML source. There are several libraries available to
-achieve this:
+achieve this, such as:
 
  * `BeautifulSoup`_ is a very popular web scraping library among Python
    programmers which constructs a Python object based on the structure of the
@@ -25,8 +25,9 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are built over the `lxml`_ library, which means they're very
-similar in speed and parsing accuracy.
+Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
+under the hood. It means Scrapy selectors are very similar in speed and
+parsing accuracy to lxml.
 
 This page explains how selectors work and describes their API which is very
 small and simple, unlike the `lxml`_ API which is much bigger because the
@@ -42,7 +43,7 @@ For a complete reference of the selectors API see
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
 .. _XPath: https://www.w3.org/TR/xpath
 .. _CSS: https://www.w3.org/TR/selectors
-
+.. _parsel: https://parsel.readthedocs.io/
 
 Using selectors
 ===============
@@ -63,21 +64,32 @@ input type::
 Constructing from text::
 
     >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
 
 Constructing from response::
 
     >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
 
 For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible::
+it's totally OK to use this shortcut when possible. By using it you can
+ensure the response body is parsed only once::
 
-    >>> response.selector.xpath('//span/text()').extract()
-    [u'good']
+    >>> response.selector.xpath('//span/text()').get()
+    'good'
 
+Querying responses using XPath and CSS is so common that responses include two
+more shortcuts: ``response.xpath()`` and ``response.css()``::
+
+    >>> response.xpath('//span/text()').get()
+    'good'
+    >>> response.css('span::text').get()
+    'good'
+
+Usually there is no need to construct Scrapy selectors manually because of
+these shortcuts.
 
 Using selectors
 ---------------
@@ -90,7 +102,7 @@ documentation server:
 
 .. _topics-selectors-htmlcode:
 
-Here's its HTML code:
+For the sake of completeness, here's its full HTML code:
 
 .. literalinclude:: ../_static/selectors-sample1.html
    :language: html
@@ -111,90 +123,179 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
 So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
 page, let's construct an XPath for selecting the text inside the title tag::
 
-    >>> response.selector.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-
-Querying responses using XPath and CSS is so common that responses include two
-convenience shortcuts: ``response.xpath()`` and ``response.css()``::
-
     >>> response.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-    >>> response.css('title::text')
-    [<Selector (text) xpath=//title/text()>]
+    [<Selector xpath='//title/text()' data='Example website'>]
+
+To actually extract the textual data, you must call the selector ``.get()``
+or ``.getall()`` methods, as follows::
+
+    >>> response.xpath('//title/text()').getall()
+    ['Example website']
+    >>> response.xpath('//title/text()').get()
+    'Example website'
+
+``.get()`` always returns a single result; if there are several matches,
+content of a first match is returned; if there are no matches, None
+is returned. ``.getall()`` returns a list with all results.
+
+Notice that CSS selectors can select text or attribute nodes using CSS3
+pseudo-elements::
+
+    >>> selector.css('title::text').get()
+    'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
 selectors. This API can be used for quickly selecting nested data::
 
-    >>> response.css('img').xpath('@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('img').xpath('@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
 
-To actually extract the textual data, you must call the selector ``.extract()``
-method, as follows::
+If you want to extract only the first matched element, you can call the
+selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
+previous Scrapy versions)::
 
-    >>> response.xpath('//title/text()').extract()
-    [u'Example website']
+    >>> response.xpath('//div[@id="images"]/a/text()').get()
+    'Name: My image 1 '
 
-If you want to extract only first matched element, you can call the selector ``.extract_first()``
+It returns ``None`` if no element was found::
 
-    >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
-    u'Name: My image 1 '
-
-It returns ``None`` if no element was found:
-
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
     True
 
-A default return value can be provided as an argument, to be used instead of ``None``:
+A default return value can be provided as an argument, to be used instead
+of ``None``:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+    >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
     'not-found'
 
-Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
 
-    >>> response.css('title::text').extract()
-    [u'Example website']
+    >>> [img.attrib['src'] for img in response.css('img')]
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+As a shortcut, ``.attrib`` is also available on SelectorList directly;
+it returns attributes for the first matching element::
+
+    >>> response.css('img').attrib['src']
+    'image1_thumb.jpg'
+
+This is most useful when only a single result is expected, e.g. when selecting
+by id, or selecting unique elements on a web page::
+
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
 Now we're going to get the base URL and some image links::
 
-    >>> response.xpath('//base/@href').extract()
-    [u'http://example.com/']
+    >>> response.xpath('//base/@href').get()
+    'http://example.com/'
 
-    >>> response.css('base::attr(href)').extract()
-    [u'http://example.com/']
+    >>> response.css('base::attr(href)').get()
+    'http://example.com/'
 
-    >>> response.xpath('//a[contains(@href, "image")]/@href').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
-    >>> response.css('a[href*=image]::attr(href)').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('a[href*=image]::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.css('a[href*=image] img::attr(src)').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+    >>> response.css('a[href*=image] img::attr(src)').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+.. _topics-selectors-css-extensions:
+
+Extensions to CSS Selectors
+---------------------------
+
+Per W3C standards, `CSS selectors`_ do not support selecting text nodes
+or attribute values.
+But selecting these is so essential in a web scraping context
+that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
+
+* to select text nodes, use ``::text``
+* to select attribute values, use ``::attr(name)`` where *name* is the
+  name of the attribute that you want the value of
+
+.. warning::
+    These pseudo-elements are Scrapy-/Parsel-specific.
+    They will most probably not work with other libraries like
+   `lxml`_ or `PyQuery`_.
+
+.. _PyQuery: https://pypi.python.org/pypi/pyquery
+
+Examples:
+
+* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
+
+    >>> response.css('title::text').get()
+    'Example website'
+
+* ``*::text`` selects all descendant text nodes of the current selector context::
+
+    >>> response.css('#images *::text').getall()
+    ['\n   ',
+     'Name: My image 1 ',
+     '\n   ',
+     'Name: My image 2 ',
+     '\n   ',
+     'Name: My image 3 ',
+     '\n   ',
+     'Name: My image 4 ',
+     '\n   ',
+     'Name: My image 5 ',
+     '\n  ']
+
+* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
+
+.. note::
+    You cannot chain these pseudo-elements. But in practice it would not
+    make much sense: text nodes do not have attributes, and attribute values
+    are string values already and do not have children nodes.
+
+.. note::
+    See also: :ref:`selecting-attributes`.
+
+
+.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors:
 
@@ -206,22 +307,65 @@ of the same type, so you can call the selection methods for those selectors
 too. Here's an example::
 
     >>> links = response.xpath('//a[contains(@href, "image")]')
-    >>> links.extract()
-    [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
-     u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
-     u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
-     u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
-     u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+    >>> links.getall()
+    ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+     '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+     '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+     '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+     '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
 
     >>> for index, link in enumerate(links):
-    ...     args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract())
-    ...     print 'Link number %d points to url %s and image %s' % args
+    ...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+    ...     print('Link number %d points to url %r and image %r' % args)
 
-    Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
-    Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg']
-    Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg']
-    Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg']
-    Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg']
+    Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+    Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+    Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+    Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+    Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+
+.. _selecting-attributes:
+
+Selecting element attributes
+----------------------------
+
+There are several ways to get a value of an attribute. First, one can use
+XPath syntax::
+
+    >>> response.xpath("//a/@href").getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+XPath syntax has a few advantages: it is a standard XPath feature, and
+``@attributes`` can be used in other parts of an XPath expression - e.g.
+it is possible to filter by attribute value.
+
+Scrapy also provides an extension to CSS selectors (``::attr(...)``)
+which allows to get attribute values::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+In addition to that, there is a ``.attrib`` property of Selector.
+You can use it if you prefer to lookup attributes in Python
+code, without using XPaths or CSS extensions::
+
+    >>> [a.attrib['href'] for a in response.css('a')]
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+This property is also available on SelectorList; it returns a dictionary
+with attributes of a first matching element. It is convenient to use when
+a selector is expected to give a single result (e.g. when selecting by element
+ID, or when selecting an unique element on a page)::
+
+    >>> response.css('base').attrib
+    {'href': 'http://example.com/'}
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
+
+``.attrib`` property of an empty SelectorList is empty::
+
+    >>> response.css('foo').attrib
+    {}
 
 Using selectors with regular expressions
 ----------------------------------------
@@ -241,8 +385,9 @@ Here's an example used to extract image names from the :ref:`HTML code
      'My image 4',
      'My image 5']
 
-There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
-named ``.re_first()``. Use it to extract just the first matching string::
+There's an additional helper reciprocating ``.get()`` (and its
+alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
+Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
@@ -266,17 +411,17 @@ it actually extracts all ``<p>`` elements from the document, not only those
 inside ``<div>`` elements::
 
     >>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
-    ...     print p.extract()
+    ...     print(p.get())
 
 This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
 
     >>> for p in divs.xpath('.//p'):  # extracts all <p> inside
-    ...     print p.extract()
+    ...     print(p.get())
 
 Another common case would be to extract all direct ``<p>`` children::
 
     >>> for p in divs.xpath('p'):
-    ...     print p.extract()
+    ...     print(p.get())
 
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
@@ -298,14 +443,14 @@ Here's an example to match an element based on its "id" attribute value,
 without hard-coding it (that was shown previously)::
 
     >>> # `$val` used in the expression, a `val` argument needs to be passed
-    >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
-    u'Name: My image 1 '
+    >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+    'Name: My image 1 '
 
 Here's another example, to find the "id" attribute of a ``<div>`` tag containing
 five ``<a>`` children (here we pass the value ``5`` as an integer)::
 
-    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
-    u'images'
+    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+    'images'
 
 All variable references must have a binding value when calling ``.xpath()``
 (otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -314,13 +459,12 @@ This is done by passing as many named arguments as necessary.
 `parsel`_, the library powering Scrapy selectors, has more details and examples
 on `XPath variables`_.
 
-.. _parsel: https://parsel.readthedocs.io/
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
 Using EXSLT extensions
 ----------------------
 
-Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions
 and come with these pre-registered namespaces to use in XPath expressions:
 
 
@@ -340,7 +484,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit::
 
     >>> from scrapy import Selector
-    >>> doc = """
+    >>> doc = u"""
     ... <div>
     ...     <ul>
     ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -352,10 +496,10 @@ Example selecting links in list item with a "class" attribute ending with a digi
     ... </div>
     ... """
     >>> sel = Selector(text=doc, type="html")
-    >>> sel.xpath('//li//@href').extract()
-    [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
-    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
-    [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+    >>> sel.xpath('//li//@href').getall()
+    ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+    ['link1.html', 'link2.html', 'link4.html', 'link5.html']
     >>>
 
 .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
@@ -372,7 +516,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from http://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = """
+    >>> doc = u"""
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@@ -424,12 +568,12 @@ with groups of itemscopes and corresponding itemprops::
     ... """
     >>> sel = Selector(text=doc, type="html")
     >>> for scope in sel.xpath('//div[@itemscope]'):
-    ...     print "current scope:", scope.xpath('@itemtype').extract()
+    ...     print("current scope:", scope.xpath('@itemtype').getall())
     ...     props = scope.xpath('''
     ...                 set:difference(./descendant::*/@itemprop,
     ...                                .//*[@itemscope]/*/@itemprop)''')
-    ...     print "    properties:", props.extract()
-    ...     print
+    ...     print("    properties: %s" % (props.getall()))
+    ...     print("")
 
     current scope: ['http://schema.org/Product']
         properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
@@ -493,27 +637,27 @@ Example::
 
 Converting a *node-set* to string::
 
-    >>> sel.xpath('//a//text()').extract() # take a peek at the node-set
-    [u'Click here to go to the ', u'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string
-    [u'Click here to go to the ']
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
 
 A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
 
-    >>> sel.xpath("//a[1]").extract() # select the first node
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").extract() # convert it to string
-    [u'Click here to go to the Next Page']
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
 
 So, using the ``.//text()`` node-set won't select anything in this case::
 
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract()
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
     []
 
 But using the ``.`` to mean the node, works::
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
 .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
 
@@ -538,7 +682,7 @@ Example::
     ....:         <li>5</li>
     ....:         <li>6</li>
     ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).extract()
+    >>> xp = lambda x: sel.xpath(x).getall()
 
 This gets all first ``<li>``  elements under whatever it is its parent::
 
@@ -578,12 +722,59 @@ you can just select by class using CSS and then switch to XPath when needed::
 
     >>> from scrapy import Selector
     >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').extract()
-    [u'2014-07-23 19:00']
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
 
 This is cleaner than using the verbose XPath trick shown above. Just remember
 to use the ``.`` in the XPath expressions that will follow.
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. These methods
+are still supported by Scrapy, there are no plans to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
 
 .. _topics-selectors-ref:
 
@@ -718,10 +909,12 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples-html:
+
 Selector examples on HTML response
 ----------------------------------
 
-Here's a couple of :class:`Selector` examples to illustrate several concepts.
+Here are some :class:`Selector` examples to illustrate several concepts.
 In all cases, we assume there is already a :class:`Selector` instantiated with
 a :class:`~scrapy.http.HtmlResponse` object like this::
 
@@ -735,20 +928,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
    returning a list of unicode strings::
 
-      sel.xpath("//h1").extract()         # this includes the h1 tag
-      sel.xpath("//h1/text()").extract()  # this excludes the h1 tag
+      sel.xpath("//h1").getall()         # this includes the h1 tag
+      sel.xpath("//h1/text()").getall()  # this excludes the h1 tag
 
 3. Iterate over all ``<p>`` tags and print their class attribute::
 
       for node in sel.xpath("//p"):
-          print node.xpath("@class").extract()
+          print(node.attrib['class'])
+
+
+.. _selector-examples-xml:
 
 Selector examples on XML response
 ---------------------------------
 
-Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already a :class:`Selector` instantiated with an
-:class:`~scrapy.http.XmlResponse` object like this::
+Here are some examples to illustrate concepts for :class:`Selector` objects
+instantiated with an :class:`~scrapy.http.XmlResponse` object::
 
       sel = Selector(xml_response)
 
@@ -761,7 +956,7 @@ assume there is already a :class:`Selector` instantiated with an
    a namespace::
 
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
-      sel.xpath("//g:price").extract()
+      sel.xpath("//g:price").getall()
 
 .. _removing-namespaces:
 
@@ -781,6 +976,20 @@ First, we open the shell with the url we want to scrape::
 
     $ scrapy shell https://github.com/blog.atom
 
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
 .. highlight:: python
 
 Once in the shell we can try selecting all ``<link>`` objects and see that it
@@ -794,8 +1003,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default
@@ -803,8 +1012,8 @@ instead of having to call it manually, this is because of two reasons, which, in
 of relevance, are:
 
 1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform for all
-   documents crawled by Scrapy
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
 
 2. There could be some cases where using namespaces is actually required, in
    case some element names clash between namespaces. These cases are very rare

From 09fd6c2a816ebec175ff3ffd945ce3263c9f1570 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 03:59:58 +0500
Subject: [PATCH 06/18] DOC unlink Firefox & Firebug sections from the tutorial
 for now.

See https://github.com/scrapy/scrapy/issues/3373 and https://github.com/scrapy/scrapy/issues/3372 for motivation.
---
 docs/intro/tutorial.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a24cf0f5b..453f5114f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -299,7 +299,8 @@ expressions`::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
+You can use your browser developer tools to inspect the HTML and come up
+with a selector (see section about :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.

From 2c48d156db7d7846d80fa379dfb33552ac5b5a85 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:01:17 +0500
Subject: [PATCH 07/18] DOC cleanup references in tutorials:

* remove unused link
* fix ReST syntax
* fix a link to regular expression docs
---
 docs/intro/tutorial.rst   | 3 +--
 docs/topics/selectors.rst | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 453f5114f..92d1065af 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -288,7 +288,7 @@ to be scraped, you can at least get **some** data.
 Besides the :meth:`~scrapy.selector.Selector.extract` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
-expressions`::
+expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
     ['Quotes to Scrape']
@@ -740,4 +740,3 @@ modeling the scraped data. If you prefer to play with an example project, check
 the :ref:`intro-examples` section.
 
 .. _JSON: https://en.wikipedia.org/wiki/JSON
-.. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 00158ecf1..cf1b0ba22 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -251,7 +251,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
 .. warning::
     These pseudo-elements are Scrapy-/Parsel-specific.
     They will most probably not work with other libraries like
-   `lxml`_ or `PyQuery`_.
+    `lxml`_ or `PyQuery`_.
 
 .. _PyQuery: https://pypi.python.org/pypi/pyquery
 

From 53da56c8dcd6edba12284ed5d125ddd1dcf6f99c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:17:55 +0500
Subject: [PATCH 08/18] TST update tests to use get/getall/attrib instead of
 extract

---
 tests/test_command_shell.py   |  2 +-
 tests/test_http_response.py   | 34 +++++++++++-----------
 tests/test_loader.py          |  4 +--
 tests/test_pipeline_crawl.py  |  2 +-
 tests/test_selector.py        | 24 +++++++--------
 tests/test_spider.py          |  8 ++---
 tests/test_utils_iterators.py | 55 ++++++++++++++++++-----------------
 7 files changed, 66 insertions(+), 63 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 3e27d6abd..36baacfbd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_response_selector_html(self):
-        xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
+        xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
         _, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
         self.assertEqual(out.strip(), b'Works')
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 820758dc9..3b90e3dac 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//title/text()").extract(),
+            response.selector.xpath("//title/text()").getall(),
             [u'Some page']
         )
         self.assertEqual(
-            response.selector.css("title::text").extract(),
+            response.selector.css("title::text").getall(),
             [u'Some page']
         )
         self.assertEqual(
@@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//title/text()").extract(),
-            response.selector.xpath("//title/text()").extract(),
+            response.xpath("//title/text()").getall(),
+            response.selector.xpath("//title/text()").getall(),
         )
         self.assertEqual(
-            response.css("title::text").extract(),
-            response.selector.css("title::text").extract(),
+            response.css("title::text").getall(),
+            response.selector.css("title::text").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
-            response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
+            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
+            response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
         )
         self.assertEqual(
             response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1).extract(),
-            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
+                pclass="content", pcount=1).getall(),
+            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
     def test_urljoin_with_base_url(self):
@@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//elem/text()").extract(),
+            response.selector.xpath("//elem/text()").getall(),
             [u'value']
         )
 
@@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//elem/text()").extract(),
-            response.selector.xpath("//elem/text()").extract(),
+            response.xpath("//elem/text()").getall(),
+            response.selector.xpath("//elem/text()").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
         )
 
         response.selector.register_namespace('s2', 'http://scrapy.org')
         self.assertEqual(
-            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s2:elem/text()").extract(),
+            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s2:elem/text()").getall(),
         )
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 3b5714058..8b58e4dbd 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
@@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_css("header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5985a6f3e..fb72c9d6d 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
                     for href in response.xpath('''
                         //table[thead/tr/th="Filename"]
                             /tbody//a/@href
-                        ''').extract()],
+                        ''').getall()],
         }
         yield item
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 526660cc8..bc4baf7ea 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
         for x in xl:
             assert isinstance(x, Selector)
 
-        self.assertEqual(sel.xpath('//input').extract(),
-                         [x.extract() for x in sel.xpath('//input')])
+        self.assertEqual(sel.xpath('//input').getall(),
+                         [x.get() for x in sel.xpath('//input')])
 
-        self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
+        self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
                          [u'a'])
-        self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+        self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
                          [u'12.0'])
 
-        self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
+        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
                          [u'xpathrules'])
-        self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+        self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
     def test_root_base_url(self):
@@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
@@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
                           [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
-        r1 = TextResponse('http://www.example.com', \
-                          body=b'<html><p>an Jos\xe9 de</p><html>', \
+        r1 = TextResponse('http://www.example.com',
+                          body=b'<html><p>an Jos\xe9 de</p><html>',
                           encoding='utf-8')
-        Selector(r1).xpath('//text()').extract()
+        Selector(r1).xpath('//text()').getall()
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 929e0fea8..f26da2334 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
 
             def parse_node(self, response, selector):
                 yield {
-                    'loc': selector.xpath('a:loc/text()').extract(),
-                    'updated': selector.xpath('b:updated/text()').extract(),
-                    'other': selector.xpath('other/@value').extract(),
-                    'custom': selector.xpath('other/@b:custom').extract(),
+                    'loc': selector.xpath('a:loc/text()').getall(),
+                    'updated': selector.xpath('b:updated/text()').getall(),
+                    'other': selector.xpath('other/@value').getall(),
+                    'custom': selector.xpath('other/@b:custom').getall(),
                 }
 
         for iterator in ('iternodes', 'xml'):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f953076b8..00eb78068 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
         for x in self.xmliter(response, 'product'):
-            attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
+            attrs.append((
+                x.attrib['id'],
+                x.xpath("name/text()").extract(),
+                x.xpath("./type/text()").extract()))
 
         self.assertEqual(attrs,
-                         [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
+                         [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
 
     def test_xmliter_unusual_node(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').extract()
+        nodenames = [e.xpath('name()').getall()
                  for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
@@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
 
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
-                attrs.append((x.xpath('@id').extract(),
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+                attrs.append((x.attrib['id'],
+                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [([u'26'], [u'-'], [u'80']),
-                              ([u'21'], [u'Ab'], [u'76']),
-                              ([u'27'], [u'A'], [u'27'])])
+                             [(u'26', [u'-'], [u'80']),
+                              (u'21', [u'Ab'], [u'76']),
+                              (u'27', [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
 
-        self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
+        self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
@@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
 
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
-        self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
-        self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
-        self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
-        self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
-        self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
-        self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
-        self.assertEqual(node.xpath('image_link/text()').extract(), [])
-        self.assertEqual(node.xpath('id/text()').extract(), [])
-        self.assertEqual(node.xpath('price/text()').extract(), [])
+        self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
+        self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
+        self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+        self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
+        self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
+        self.assertEqual(node.xpath('image_link/text()').getall(), [])
+        self.assertEqual(node.xpath('id/text()').getall(), [])
+        self.assertEqual(node.xpath('price/text()').getall(), [])
 
     def test_xmliter_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            next(self.xmliter(response, 'item')).extract(),
+            next(self.xmliter(response, 'item')).get(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
@@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
 
         namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
         body = b"""\
@@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
         my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
 
         node = next(my_iter)
-        self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
-        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
-        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
+        self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
+        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
+        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
 
         my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
 
         node = next(my_iter)
-        self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+        self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
 
     def test_xmliter_objtype_exception(self):
         i = self.xmliter(42, 'product')

From 8c29be606c1be71059b454e41b87354e22569423 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:18:29 +0500
Subject: [PATCH 09/18] update spider templates to use .get

---
 scrapy/templates/spiders/crawl.tmpl   | 10 +++++-----
 scrapy/templates/spiders/xmlfeed.tmpl | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 802cb88a1..878425125 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -14,8 +14,8 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        i = {}
-        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
-        #i['name'] = response.xpath('//div[@id="name"]').extract()
-        #i['description'] = response.xpath('//div[@id="description"]').extract()
-        return i
+        item = {}
+        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+        #item['name'] = response.xpath('//div[@id="name"]').get()
+        #item['description'] = response.xpath('//div[@id="description"]').get()
+        return item
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 7c2ff8850..863c9772f 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
     itertag = 'item' # change it accordingly
 
     def parse_node(self, response, selector):
-        i = {}
-        #i['url'] = selector.select('url').extract()
-        #i['name'] = selector.select('name').extract()
-        #i['description'] = selector.select('description').extract()
-        return i
+        item = {}
+        #item['url'] = selector.select('url').get()
+        #item['name'] = selector.select('name').get()
+        #item['description'] = selector.select('description').get()
+        return item

From 460f0f045141f73dcc6d7a2309b7e2f0c2a492b4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:32 +0500
Subject: [PATCH 10/18] [backwards incompatible] switch ItemLoader from
 .extract to .getall.

This change is backwards incompatible if ItemLoader is used with a custom Selector
subclass which overrides .extract without overriding .getall.
---
 scrapy/loader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index e73413318..a7c75a46a 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -181,7 +181,7 @@ class ItemLoader(object):
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
+        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
 
     def add_css(self, field_name, css, *processors, **kw):
         values = self._get_cssvalues(css, **kw)
@@ -198,6 +198,6 @@ class ItemLoader(object):
     def _get_cssvalues(self, csss, **kw):
         self._check_selector_method()
         csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).extract() for css in csss)
+        return flatten(self.selector.css(css).getall() for css in csss)
 
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

From 12e42bbe06bea5a0f86e95630b184e86b41c95c7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:55 +0500
Subject: [PATCH 11/18] switch SgmlLinkExtractor to .getall

---
 scrapy/linkextractors/sgml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index f4ca4262a..5fa6b771c 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
             base_url = get_base_url(response)
             body = u''.join(f
                             for x in self.restrict_xpaths
-                            for f in response.xpath(x).extract()
+                            for f in response.xpath(x).getall()
                             ).encode(response.encoding, errors='xmlcharrefreplace')
         else:
             body = response.body

From afce9716fabff6169dc9cc146a2589ebf4c1ed13 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:25:43 +0500
Subject: [PATCH 12/18] DOC mention .attrib in the tutorial

---
 docs/intro/tutorial.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 92d1065af..07fb4807f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -520,6 +520,12 @@ like this::
     >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
+There is also an ``attrib`` property available
+(see :ref:`selecting-attributes` for more)::
+
+    >>> response.css('li.next a').attrib['href']
+    '/page/2'
+
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
 

From bdcc045f62a34d1655a4ead4e0b38d6fe97a08fc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:27:21 +0500
Subject: [PATCH 13/18] DOC switch from .extract to get/getall API in docs

Also, response.urljoin is added in a few places, for robustness.
---
 docs/intro/tutorial.rst | 22 +++++++++++-----------
 docs/topics/shell.rst   |  4 ++--
 docs/topics/spiders.rst | 14 +++++++-------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 07fb4807f..46e84b21c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -254,7 +254,7 @@ data.
 
 To extract the text from the title above, you can do::
 
-    >>> response.css('title::text').extract()
+    >>> response.css('title::text').getall()
     ['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
@@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside
 ``<title>`` element.  If we don't specify ``::text``, we'd get the full title
 element, including its tags::
 
-    >>> response.css('title').extract()
+    >>> response.css('title').getall()
     ['<title>Quotes to Scrape</title>']
 
-The other thing is that the result of calling ``.extract()`` is a list, because
-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
-you know you just want the first result, as in this case, you can do::
+The other thing is that the result of calling ``.getall()`` is a list: it is
+possible that a selector returns more than one result, so we extract them all.
+When you know you just want the first result, as in this case, you can do::
 
     >>> response.css('title::text').get()
     'Quotes to Scrape'
@@ -392,10 +392,10 @@ using the ``quote`` object we just created::
     >>> author
     'Albert Einstein'
 
-Given that the tags are a list of strings, we can use the ``.extract()`` method
+Given that the tags are a list of strings, we can use the ``.getall()`` method
 to get all of them::
 
-    >>> tags = quote.css("div.tags a.tag::text").extract()
+    >>> tags = quote.css("div.tags a.tag::text").getall()
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
@@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary::
     >>> for quote in response.css("div.quote"):
     ...     text = quote.css("span.text::text").get()
     ...     author = quote.css("small.author::text").get()
-    ...     tags = quote.css("div.tags a.tag::text").extract()
+    ...     tags = quote.css("div.tags a.tag::text").getall()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
     {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
@@ -438,7 +438,7 @@ in the callback, as you can see below::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
 If you run this spider, it will output the extracted data with the log::
@@ -543,7 +543,7 @@ page, extracting data from it::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
@@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can use
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('span small::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 9de6abef7..68a0b19b5 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -184,8 +184,8 @@ After that, we can start playing with the objects::
 
     >>> fetch("https://reddit.com")
 
-    >>> response.xpath('//title/text()').extract()
-    ['reddit: the front page of the internet']
+    >>> response.xpath('//title/text()').get()
+    'reddit: the front page of the internet'
 
     >>> request = request.replace(method="POST")
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 4505b7315..a08dc30f2 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::
         ]
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield {"title": h3}
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
 to give data more structure you can use :ref:`topics-items`::
@@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::
             yield scrapy.Request('http://www.example.com/3.html', self.parse)
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield MyItem(title=h3)
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 .. _spiderargs:
 
@@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example::
         itertag = 'item'
 
         def parse_node(self, response, node):
-            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
+            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
 
             item = TestItem()
             item['id'] = node.xpath('@id').get()

From 7fdfdb7fa244ed44274aaa9750efec1c333eedd1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 17:34:44 +0500
Subject: [PATCH 14/18] DOC reorganize selectors tutorial, port more topics
 from parsel docs, adjust wording in the introduction

---
 docs/topics/selectors.rst | 514 +++++++++++++++++++++-----------------
 1 file changed, 281 insertions(+), 233 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index cf1b0ba22..2db982b37 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -25,17 +25,14 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
-under the hood. It means Scrapy selectors are very similar in speed and
-parsing accuracy to lxml.
+.. note::
+    Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of
+    this wrapper is to provide better integration with Scrapy Response objects.
 
-This page explains how selectors work and describes their API which is very
-small and simple, unlike the `lxml`_ API which is much bigger because the
-`lxml`_ library can be used for many other tasks, besides selecting markup
-documents.
-
-For a complete reference of the selectors API see
-:ref:`Selector reference <topics-selectors-ref>`
+    `parsel`_ is a stand-alone web scraping library which can be used without
+    Scrapy. It uses `lxml`_ library under the hood, and implements an
+    easy API on top of lxml API. It means Scrapy selectors are very similar
+    in speed and parsing accuracy to lxml.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
@@ -73,9 +70,8 @@ Constructing from response::
     >>> Selector(response=response).xpath('//span/text()').get()
     'good'
 
-For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible. By using it you can
-ensure the response body is parsed only once::
+For convenience, response objects expose a selector on `.selector` attribute.
+By using it you can ensure the response body is parsed only once::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -88,8 +84,10 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
-Usually there is no need to construct Scrapy selectors manually because of
-these shortcuts.
+Usually there is no need to construct Scrapy selectors manually:
+``response`` object is available in Spider callbacks, so in most cases
+it is more convenient to use ``response.css()`` and ``response.xpath()``
+shortcuts.
 
 Using selectors
 ---------------
@@ -392,6 +390,71 @@ Use it to extract just the first matching string::
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts
+and tutorials are using them as well. These methods are still supported
+by Scrapy, there are **no plans** to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
+
+.. _topics-selectors-xpaths:
+
+Working with XPaths
+===================
+
+Here are some tips which may help you to use XPath with Scrapy selectors
+effectively. If you are not much familiar with XPath yet,
+you may want to take a look first at this `XPath tutorial`_.
+
+.. note::
+    Some of the tips are based on `this post from ScrapingHub's blog`_.
+
+.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+
+
 .. _topics-selectors-relative-xpaths:
 
 Working with relative XPaths
@@ -428,6 +491,114 @@ XPath specification.
 
 .. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
+When querying by class, consider using CSS
+------------------------------------------
+
+Because an element can contain multiple CSS classes, the XPath way to select elements
+by class is the rather verbose::
+
+    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
+
+If you use ``@class='someclass'`` you may end up missing elements that have
+other classes, and if you just use ``contains(@class, 'someclass')`` to make up
+for that you may end up with more elements that you want, if they have a different
+class name that shares the string ``someclass``.
+
+As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
+you can just select by class using CSS and then switch to XPath when needed::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
+
+This is cleaner than using the verbose XPath trick shown above. Just remember
+to use the ``.`` in the XPath expressions that will follow.
+
+Beware of the difference between //node[1] and (//node)[1]
+----------------------------------------------------------
+
+``//node[1]`` selects all the nodes occurring first under their respective parents.
+
+``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text="""
+    ....:     <ul class="list">
+    ....:         <li>1</li>
+    ....:         <li>2</li>
+    ....:         <li>3</li>
+    ....:     </ul>
+    ....:     <ul class="list">
+    ....:         <li>4</li>
+    ....:         <li>5</li>
+    ....:         <li>6</li>
+    ....:     </ul>""")
+    >>> xp = lambda x: sel.xpath(x).getall()
+
+This gets all first ``<li>``  elements under whatever it is its parent::
+
+    >>> xp("//li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element in the whole document::
+
+    >>> xp("(//li)[1]")
+    ['<li>1</li>']
+
+This gets all first ``<li>``  elements under an ``<ul>``  parent::
+
+    >>> xp("//ul/li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
+
+    >>> xp("(//ul/li)[1]")
+    ['<li>1</li>']
+
+Using text nodes in a condition
+-------------------------------
+
+When you need to use the text content as argument to an `XPath string function`_,
+avoid using ``.//text()`` and use just ``.`` instead.
+
+This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
+And when a node-set is converted to a string, which happens when it is passed as argument to
+a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+
+Converting a *node-set* to string::
+
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
+
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
+
+So, using the ``.//text()`` node-set won't select anything in this case::
+
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    []
+
+But using the ``.`` to mean the node, works::
+
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+
 .. _topics-selectors-xpath-variables:
 
 Variables in XPath expressions
@@ -461,6 +632,69 @@ on `XPath variables`_.
 
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
+
+.. _removing-namespaces:
+
+Removing namespaces
+-------------------
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names, to write more
+simple/convenient XPaths. You can use the
+:meth:`Selector.remove_namespaces` method for that.
+
+Let's show an example that illustrates this with GitHub blog atom feed.
+
+.. highlight:: sh
+
+First, we open the shell with the url we want to scrape::
+
+    $ scrapy shell https://github.com/blog.atom
+
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
+.. highlight:: python
+
+Once in the shell we can try selecting all ``<link>`` objects and see that it
+doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+
+    >>> response.xpath("//link")
+    []
+
+But once we call the :meth:`Selector.remove_namespaces` method, all
+nodes can be accessed directly by their names::
+
+    >>> response.selector.remove_namespaces()
+    >>> response.xpath("//link")
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     ...
+
+If you wonder why the namespace removal procedure isn't always called by default
+instead of having to call it manually, this is because of two reasons, which, in order
+of relevance, are:
+
+1. Removing namespaces requires to iterate and modify all nodes in the
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
+
+2. There could be some cases where using namespaces is actually required, in
+   case some element names clash between namespaces. These cases are very rare
+   though.
+
+
 Using EXSLT extensions
 ----------------------
 
@@ -606,174 +840,44 @@ inside another ``itemscope``.
 .. _regular expressions: http://exslt.org/regexp/index.html
 .. _set manipulation: http://exslt.org/set/index.html
 
+Other XPath extensions
+----------------------
 
-Some XPath tips
----------------
+Scrapy selectors also provide a sorely missed XPath extension function
+``has-class`` that returns ``True`` for nodes that have all of the specified
+HTML classes.
 
-Here are some tips that you may find useful when using XPath
-with Scrapy selectors, based on `this post from ScrapingHub's blog`_.
-If you are not much familiar with XPath yet,
-you may want to take a look first at this `XPath tutorial`_.
+.. highlight:: html
 
+For the following HTML::
 
-.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+    <p class="foo bar-baz">First</p>
+    <p class="foo">Second</p>
+    <p class="bar">Third</p>
+    <p>Fourth</p>
 
+.. highlight:: python
 
-Using text nodes in a condition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can use it like this::
 
-When you need to use the text content as argument to an `XPath string function`_,
-avoid using ``.//text()`` and use just ``.`` instead.
-
-This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
-And when a node-set is converted to a string, which happens when it is passed as argument to
-a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
-
-Converting a *node-set* to string::
-
-    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
-    ['Click here to go to the ', 'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
-    ['Click here to go to the ']
-
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
-
-    >>> sel.xpath("//a[1]").getall() # select the first node
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").getall() # convert it to string
-    ['Click here to go to the Next Page']
-
-So, using the ``.//text()`` node-set won't select anything in this case::
-
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    >>> response.xpath('//p[has-class("foo")]')
+    [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+     <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar-baz")]')
+    [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar")]')
     []
 
-But using the ``.`` to mean the node, works::
+So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
+``p.foo.bar-baz``.  Please note, that it is slower in most of the cases,
+because it's a pure-Python function that's invoked for every node in question
+whereas the CSS lookup is translated into XPath and thus runs more efficiently,
+so performance-wise its uses are limited to situations that are not easily
+described with CSS selectors.
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+Parsel also simplifies adding your own XPath extensions.
 
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
-
-Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``//node[1]`` selects all the nodes occurring first under their respective parents.
-
-``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text="""
-    ....:     <ul class="list">
-    ....:         <li>1</li>
-    ....:         <li>2</li>
-    ....:         <li>3</li>
-    ....:     </ul>
-    ....:     <ul class="list">
-    ....:         <li>4</li>
-    ....:         <li>5</li>
-    ....:         <li>6</li>
-    ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).getall()
-
-This gets all first ``<li>``  elements under whatever it is its parent::
-
-    >>> xp("//li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element in the whole document::
-
-    >>> xp("(//li)[1]")
-    ['<li>1</li>']
-
-This gets all first ``<li>``  elements under an ``<ul>``  parent::
-
-    >>> xp("//ul/li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
-
-    >>> xp("(//ul/li)[1]")
-    ['<li>1</li>']
-
-When querying by class, consider using CSS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Because an element can contain multiple CSS classes, the XPath way to select elements
-by class is the rather verbose::
-
-    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
-
-If you use ``@class='someclass'`` you may end up missing elements that have
-other classes, and if you just use ``contains(@class, 'someclass')`` to make up
-for that you may end up with more elements that you want, if they have a different
-class name that shares the string ``someclass``.
-
-As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').getall()
-    ['2014-07-23 19:00']
-
-This is cleaner than using the verbose XPath trick shown above. Just remember
-to use the ``.`` in the XPath expressions that will follow.
-
-.. _old-extraction-api:
-
-extract() and extract_first()
------------------------------
-
-If you're a long-time Scrapy user, you're probably familiar
-with ``.extract()`` and ``.extract_first()`` selector methods. These methods
-are still supported by Scrapy, there are no plans to deprecate them.
-
-However, Scrapy usage docs are now written using ``.get()`` and
-``.getall()`` methods. We feel that these new methods result in a more concise
-and readable code.
-
-The following examples show how these methods map to each other.
-
-1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
-
-     >>> response.css('a::attr(href)').get()
-     'image1.html'
-     >>> response.css('a::attr(href)').extract_first()
-     'image1.html'
-
-2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
-
-     >>> response.css('a::attr(href)').getall()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-     >>> response.css('a::attr(href)').extract()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
-
-     >>> response.css('a::attr(href)')[0].get()
-     'image1.html'
-     >>> response.css('a::attr(href)')[0].extract()
-     'image1.html'
-
-4. For consistency, there is also ``Selector.getall()``, which returns a list::
-
-    >>> response.css('a::attr(href)')[0].getall()
-    ['image1.html']
-
-So, the main difference is that output of ``.get()`` and ``.getall()`` methods
-is more predictable: ``.get()`` always returns a single result, ``.getall()``
-always returns a list of all extracted results. With ``.extract()`` method
-it was not always obvious if a result is a list or not; to get a single
-result either ``.extract()`` or ``.extract_first()`` should be called.
+.. autofunction:: parsel.xpathfuncs.set_xpathfunc
 
 
 .. _topics-selectors-ref:
@@ -909,6 +1013,11 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples:
+
+Examples
+========
+
 .. _selector-examples-html:
 
 Selector examples on HTML response
@@ -958,65 +1067,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
       sel.xpath("//g:price").getall()
 
-.. _removing-namespaces:
-
-Removing namespaces
--------------------
-
-When dealing with scraping projects, it is often quite convenient to get rid of
-namespaces altogether and just work with element names, to write more
-simple/convenient XPaths. You can use the
-:meth:`Selector.remove_namespaces` method for that.
-
-Let's show an example that illustrates this with GitHub blog atom feed.
-
-.. highlight:: sh
-
-First, we open the shell with the url we want to scrape::
-
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
-
-This is how the file starts::
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
-      ...
-
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
-
-.. highlight:: python
-
-Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
-
-    >>> response.xpath("//link")
-    []
-
-But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
-
-    >>> response.selector.remove_namespaces()
-    >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     ...
-
-If you wonder why the namespace removal procedure isn't always called by default
-instead of having to call it manually, this is because of two reasons, which, in order
-of relevance, are:
-
-1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform by default
-   for all documents crawled by Scrapy
-
-2. There could be some cases where using namespaces is actually required, in
-   case some element names clash between namespaces. These cases are very rare
-   though.
-
 .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799

From dc95ecbe25a5921f902fbccc93d50f7682cefda2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 12 Sep 2018 18:36:25 +0500
Subject: [PATCH 15/18] DOC use autodocs for selectors; document more methods
 and attributes; suggest get/getall

---
 docs/topics/selectors.rst  | 114 +++++++++----------------------------
 scrapy/selector/unified.py |  33 +++++++++++
 2 files changed, 60 insertions(+), 87 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 2db982b37..95d6a1a04 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -891,127 +891,67 @@ Built-in Selectors reference
 Selector objects
 ----------------
 
-.. class:: Selector(response=None, text=None, type=None)
+.. autoclass:: Selector
 
-  An instance of :class:`Selector` is a wrapper over response to select
-  certain parts of its content.
-
-  ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
-  :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
-  extracting data.
-
-  ``text`` is a unicode string or utf-8 encoded text for cases when a
-  ``response`` isn't available. Using ``text`` and ``response`` together is
-  undefined behavior.
-
-  ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
-
-    If ``type`` is ``None``, the selector automatically chooses the best type
-    based on ``response`` type (see below), or defaults to ``"html"`` in case it
-    is used together with ``text``.
-
-    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follows:
-
-        * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
-        * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
-        * ``"html"`` for anything else
-
-   Otherwise, if ``type`` is set, the selector type will be forced and no
-   detection will occur.
-
-  .. method:: xpath(query)
-
-      Find nodes matching the xpath ``query`` and return the result as a
-      :class:`SelectorList` instance with all elements flattened. List
-      elements implement :class:`Selector` interface too.
-
-      ``query`` is a string containing the XPATH query to apply.
+  .. automethod:: xpath
 
       .. note::
 
           For convenience, this method can be called as ``response.xpath()``
 
-  .. method:: css(query)
-
-      Apply the given CSS selector and return a :class:`SelectorList` instance.
-
-      ``query`` is a string containing the CSS selector to apply.
-
-      In the background, CSS queries are translated into XPath queries using
-      `cssselect`_ library and run ``.xpath()`` method.
+  .. automethod:: css
 
       .. note::
 
-          For convenience this method can be called as ``response.css()``
+          For convenience, this method can be called as ``response.css()``
 
-  .. method:: extract()
+  .. automethod:: get
 
-     Serialize and return the matched nodes as a list of unicode strings.
-     Percent encoded content is unquoted.
+     See also: :ref:`old-extraction-api`
 
-  .. method:: re(regex)
+  .. autoattribute:: attrib
 
-     Apply the given regex and return a list of unicode strings with the
-     matches.
+     See also: :ref:`selecting-attributes`.
 
-     ``regex`` can be either a compiled regular expression or a string which
-     will be compiled to a regular expression using ``re.compile(regex)``
+  .. automethod:: re
 
-    .. note::
+  .. automethod:: re_first
 
-        Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``&lt;`` and ``&amp;``).
+  .. automethod:: register_namespace
 
-  .. method:: register_namespace(prefix, uri)
+  .. automethod:: remove_namespaces
 
-     Register the given namespace to be used in this :class:`Selector`.
-     Without registering namespaces you can't select or extract data from
-     non-standard namespaces. See examples below.
+  .. automethod:: __bool__
 
-  .. method:: remove_namespaces()
-
-     Remove all namespaces, allowing to traverse the document using
-     namespace-less xpaths. See example below.
-
-  .. method:: __nonzero__()
-
-     Returns ``True`` if there is any real content selected or ``False``
-     otherwise.  In other words, the boolean value of a :class:`Selector` is
-     given by the contents it selects.
+  .. automethod:: getall
 
+     This method is added to Selector for consistency; it is more useful
+     with SelectorList. See also: :ref:`old-extraction-api`
 
 SelectorList objects
 --------------------
 
-.. class:: SelectorList
+.. autoclass:: SelectorList
 
-   The :class:`SelectorList` class is a subclass of the builtin ``list``
-   class, which provides a few additional methods.
+   .. automethod:: xpath
 
-   .. method:: xpath(query)
+   .. automethod:: css
 
-       Call the ``.xpath()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+   .. automethod:: getall
 
-       ``query`` is the same argument as the one in :meth:`Selector.xpath`
+      See also: :ref:`old-extraction-api`
 
-   .. method:: css(query)
+   .. automethod:: get
 
-       Call the ``.css()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+      See also: :ref:`old-extraction-api`
 
-       ``query`` is the same argument as the one in :meth:`Selector.css`
+   .. automethod:: re
 
-   .. method:: extract()
+   .. automethod:: re_first
 
-       Call the ``.extract()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
-
-   .. method:: re()
-
-       Call the ``.re()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
+   .. autoattribute:: attrib
 
+      See also: :ref:`selecting-attributes`.
 
 .. _selector-examples:
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 64cb0232c..8f6cb1d79 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -27,6 +27,10 @@ def _response_from_text(text, st):
 
 
 class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
+    """
+    The :class:`SelectorList` class is a subclass of the builtin ``list``
+    class, which provides a few additional methods.
+    """
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]
@@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
 
 
 class Selector(_ParselSelector, object_ref):
+    """
+    An instance of :class:`Selector` is a wrapper over response to select
+    certain parts of its content.
+
+    ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
+    :class:`~scrapy.http.XmlResponse` object that will be used for selecting
+    and extracting data.
+
+    ``text`` is a unicode string or utf-8 encoded text for cases when a
+    ``response`` isn't available. Using ``text`` and ``response`` together is
+    undefined behavior.
+
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+    or ``None`` (default).
+
+    If ``type`` is ``None``, the selector automatically chooses the best type
+    based on ``response`` type (see below), or defaults to ``"html"`` in case it
+    is used together with ``text``.
+
+    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
+    inferred from the response type as follows:
+
+    * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
+    * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+    * ``"html"`` for anything else
+
+    Otherwise, if ``type`` is set, the selector type will be forced and no
+    detection will occur.
+    """
 
     __slots__ = ['response']
     selectorlist_cls = SelectorList

From 9db21e55028feec932136e74dad0c0f0dbe7f436 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 15 Sep 2018 02:43:37 +0500
Subject: [PATCH 16/18] DOC fix remove_namespaces example

See https://github.com/scrapy/parsel/pull/119
---
 docs/topics/selectors.rst | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 95d6a1a04..20c3fff3c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -643,27 +643,30 @@ namespaces altogether and just work with element names, to write more
 simple/convenient XPaths. You can use the
 :meth:`Selector.remove_namespaces` method for that.
 
-Let's show an example that illustrates this with GitHub blog atom feed.
+Let's show an example that illustrates this with the Python Insider blog atom feed.
 
 .. highlight:: sh
 
 First, we open the shell with the url we want to scrape::
 
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
+    $ scrapy shell https://feeds.feedburner.com/PythonInsider
 
 This is how the file starts::
 
     <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
+    <?xml-stylesheet ...
+    <feed xmlns="http://www.w3.org/2005/Atom"
+          xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/"
+          xmlns:blogger="http://schemas.google.com/blogger/2008"
+          xmlns:georss="http://www.georss.org/georss"
+          xmlns:gd="http://schemas.google.com/g/2005"
+          xmlns:thr="http://purl.org/syndication/thread/1.0"
+          xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
       ...
 
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+You can see several namespace declarations including a default
+"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for
+"http://schemas.google.com/g/2005".
 
 .. highlight:: python
 
@@ -678,8 +681,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+     <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default

From 2c3b2158c99953823500f275ccc5206c11c9a811 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:02:17 +0500
Subject: [PATCH 17/18] DOC address @stummjr's review comments

* fixed several small issues
* re-written "Creating Selectors" section
* fixed remaining .extract usage in tests
---
 docs/intro/tutorial.rst       |  6 ++--
 docs/topics/selectors.rst     | 52 +++++++++++++++++++----------------
 tests/test_utils_iterators.py |  4 +--
 3 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 46e84b21c..ad17ef096 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -285,9 +285,9 @@ There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
-Besides the :meth:`~scrapy.selector.Selector.extract` and
+Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
 expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
@@ -649,7 +649,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).get().strip()
+                return response.css(query).get(default='').strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 20c3fff3c..68913c697 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -50,28 +50,8 @@ Constructing selectors
 
 .. highlight:: python
 
-Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing **text** or :class:`~scrapy.http.TextResponse`
-object. It automatically chooses the best parsing rules (XML vs HTML) based on
-input type::
-
-    >>> from scrapy.selector import Selector
-    >>> from scrapy.http import HtmlResponse
-
-Constructing from text::
-
-    >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').get()
-    'good'
-
-Constructing from response::
-
-    >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').get()
-    'good'
-
-For convenience, response objects expose a selector on `.selector` attribute.
-By using it you can ensure the response body is parsed only once::
+Response objects expose a :class:`~scrapy.selector.Selector` instance
+on ``.selector`` attribute::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -84,10 +64,34 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
+Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
+constructed by passing either :class:`~scrapy.http.TextResponse` object or
+markup as an unicode string (in ``text`` argument).
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
-shortcuts.
+shortcuts. By using ``response.selector`` or one of these shortcuts
+you can also ensure the response body is parsed only once.
+
+But if required, it is possible to use ``Selector`` directly.
+Constructing from text::
+
+    >>> from scrapy.selector import Selector
+    >>> body = '<html><body><span>good</span></body></html>'
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
+
+Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
+:class:`~scrapy.http.TextResponse` subclasses::
+
+    >>> from scrapy.selector import Selector
+    >>> from scrapy.http import HtmlResponse
+    >>> response = HtmlResponse(url='http://example.com', body=body)
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
+
+``Selector`` automatically chooses the best parsing rules
+(XML vs HTML) based on input type.
 
 Using selectors
 ---------------
@@ -139,7 +143,7 @@ is returned. ``.getall()`` returns a list with all results.
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 
-    >>> selector.css('title::text').get()
+    >>> response.css('title::text').get()
     'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 00eb78068..2d845697e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -32,8 +32,8 @@ class XmliterTestCase(unittest.TestCase):
         for x in self.xmliter(response, 'product'):
             attrs.append((
                 x.attrib['id'],
-                x.xpath("name/text()").extract(),
-                x.xpath("./type/text()").extract()))
+                x.xpath("name/text()").getall(),
+                x.xpath("./type/text()").getall()))
 
         self.assertEqual(attrs,
                          [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])

From ffbd33edac0367e9f975b9863e0c31e1c2b72ebc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:03:35 +0500
Subject: [PATCH 18/18] DOC mention gotcha with `foo::text` selector and empty
 `foo` elements

also, move "Selecting attributes" reference closer to `a::atr(href)` example
---
 docs/topics/selectors.rst | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 68913c697..9dced7473 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -279,6 +279,19 @@ Examples:
      'Name: My image 5 ',
      '\n  ']
 
+* ``foo::text`` returns no results if ``foo`` element exists, but contains
+  no text (i.e. text is empty)::
+
+    >>> response.css('img::text').getall()
+    []
+
+  This means ``.css('foo::text').get()`` could return None even if an element
+  exists. Use ``default=''`` if you always want a string::
+
+    >>> response.css('img::text').get()
+    >>> response.css('img::text').get(default='')
+    ''
+
 * ``a::attr(href)`` selects the *href* attribute value of descendant links::
 
     >>> response.css('a::attr(href)').getall()
@@ -288,15 +301,14 @@ Examples:
      'image4.html',
      'image5.html']
 
+.. note::
+    See also: :ref:`selecting-attributes`.
+
 .. note::
     You cannot chain these pseudo-elements. But in practice it would not
     make much sense: text nodes do not have attributes, and attribute values
     are string values already and do not have children nodes.
 
-.. note::
-    See also: :ref:`selecting-attributes`.
-
-
 .. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors: