From fb7d4cbce379c4e3fac2ca89b6e0772d0c690935 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 16:08:26 +0300
Subject: [PATCH 01/44] Add error handling in contracts

---
 scrapy/contracts/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 5eaee3d11..18014e290 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -41,7 +41,11 @@ class ContractsManager(object):
         requests = []
         for method in self.tested_methods_from_spidercls(type(spider)):
             bound_method = spider.__getattribute__(method)
-            requests.append(self.from_method(bound_method, results))
+            try:
+                requests.append(self.from_method(bound_method, results))
+            except:
+                case = _create_testcase(bound_method, 'contract')
+                results.addError(case, sys.exc_info())
 
         return requests
 

From ebbde57eca310d3c5c0f530a2e668572dde4d952 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 17:50:56 +0300
Subject: [PATCH 02/44] Add custom contracts tests

---
 tests/test_contracts.py | 55 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 1cea2afb7..078ef6e0d 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,7 +5,7 @@ from twisted.trial import unittest
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
-from scrapy.contracts import ContractsManager
+from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
     ReturnsContract,
@@ -22,6 +22,21 @@ class ResponseMock(object):
     url = 'http://scrapy.org'
 
 
+class CustomSuccessContract(Contract):
+    name = 'custom_success_contract'
+
+    def adjust_request_args(self, args):
+        args['url'] = 'http://scrapy.org'
+        return args
+
+
+class CustomFailContract(Contract):
+    name = 'custom_fail_contract'
+
+    def adjust_request_args(self, args):
+        raise TypeError('Error in adjust_request_args')
+
+
 class TestSpider(Spider):
     name = 'demo_spider'
 
@@ -99,8 +114,34 @@ class TestSpider(Spider):
         pass
 
 
+class CustomContractSuccessSpider(Spider):
+    name = 'custom_contract_success_spider'
+
+    def parse(self, response):
+        """
+        @custom_success_contract
+        """
+        pass
+
+
+class CustomContractFailSpider(Spider):
+    name = 'custom_contract_fail_spider'
+
+    def parse(self, response):
+        """
+        @custom_fail_contract
+        """
+        pass
+
+
 class ContractsManagerTest(unittest.TestCase):
-    contracts = [UrlContract, ReturnsContract, ScrapesContract]
+    contracts = [
+        UrlContract,
+        ReturnsContract,
+        ScrapesContract,
+        CustomSuccessContract,
+        CustomFailContract
+    ]
 
     def setUp(self):
         self.conman = ContractsManager(self.contracts)
@@ -114,6 +155,9 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertTrue(self.results.failures)
         self.assertFalse(self.results.errors)
 
+    def should_error(self):
+        self.assertTrue(self.results.errors)
+
     def test_contracts(self):
         spider = TestSpider()
 
@@ -185,3 +229,10 @@ class ContractsManagerTest(unittest.TestCase):
                 self.results)
         request.callback(response)
         self.should_fail()
+
+    def test_custom_contracts(self):
+        self.conman.from_spider(CustomContractSuccessSpider(), self.results)
+        self.should_succeed()
+
+        self.conman.from_spider(CustomContractFailSpider(), self.results)
+        self.should_error()

From 8fc017d345af1e6fa53334fc9df92ebd8f9dc32e Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 19:25:33 +0300
Subject: [PATCH 03/44] Add dont_filter to ContractsManager requests

---
 scrapy/contracts/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 8315d21d2..2569ab151 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -50,7 +50,12 @@ class ContractsManager(object):
         if contracts:
             # calculate request args
             args, kwargs = get_spec(Request.__init__)
+
+            # Don't filter requests to allow
+            # testing different callbacks on the same URL.
+            kwargs['dont_filter'] = True
             kwargs['callback'] = method
+
             for contract in contracts:
                 kwargs = contract.adjust_request_args(kwargs)
 

From b4b1e4834376e8565d727a1a5087cae31450f931 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 22:18:43 +0300
Subject: [PATCH 04/44] Add ability to use FormRequest in contracts

---
 docs/topics/contracts.rst    |  7 +++++--
 scrapy/contracts/__init__.py |  7 ++++++-
 tests/test_contracts.py      | 26 ++++++++++++++++++++++++--
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ba1421c42..cac52042a 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -86,8 +86,11 @@ override three methods:
     .. method:: Contract.adjust_request_args(args)
 
         This receives a ``dict`` as an argument containing default arguments
-        for :class:`~scrapy.http.Request` object. Must return the same or a
-        modified version of it.
+        for request object. If ``formdata`` is in ``args``, then
+        :class:`~scrapy.http.FormRequest` object is created,
+        otherwise :class:`~scrapy.http.Request` is used.
+
+        Must return the same or a modified version of it.
 
     .. method:: Contract.pre_process(response)
 
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 8315d21d2..ca2a8d384 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -3,6 +3,7 @@ import re
 from functools import wraps
 from unittest import TestCase
 
+from scrapy import FormRequest
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
@@ -57,7 +58,11 @@ class ContractsManager(object):
             # create and prepare request
             args.remove('self')
             if set(args).issubset(set(kwargs)):
-                request = Request(**kwargs)
+                if 'formdata' in kwargs:
+                    kwargs['method'] = 'POST'
+                    request = FormRequest(**kwargs)
+                else:
+                    request = Request(**kwargs)
 
                 # execute pre and post hooks in order
                 for contract in reversed(contracts):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b07cbee1e..f2085f711 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -3,11 +3,12 @@ from unittest import TextTestResult
 from twisted.python import failure
 from twisted.trial import unittest
 
+from scrapy import FormRequest
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
-from scrapy.contracts import ContractsManager
+from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
     ReturnsContract,
@@ -24,6 +25,14 @@ class ResponseMock(object):
     url = 'http://scrapy.org'
 
 
+class CustomFormContract(Contract):
+    name = 'custom_form'
+
+    def adjust_request_args(self, args):
+        args['formdata'] = {'name': 'scrapy'}
+        return args
+
+
 class TestSpider(Spider):
     name = 'demo_spider'
 
@@ -100,9 +109,16 @@ class TestSpider(Spider):
         """
         pass
 
+    def custom_form(self, response):
+        """
+        @url http://scrapy.org
+        @custom_form
+        """
+        pass
+
 
 class ContractsManagerTest(unittest.TestCase):
-    contracts = [UrlContract, ReturnsContract, ScrapesContract]
+    contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]
 
     def setUp(self):
         self.conman = ContractsManager(self.contracts)
@@ -202,3 +218,9 @@ class ContractsManagerTest(unittest.TestCase):
 
         self.assertFalse(self.results.failures)
         self.assertTrue(self.results.errors)
+
+    def test_form_contract(self):
+        spider = TestSpider()
+        request = self.conman.from_method(spider.custom_form, self.results)
+        self.assertEqual(request.method, 'POST')
+        self.assertIsInstance(request, FormRequest)

From 4de493efdd80d8ff85a78009dcc120ac46b9c55c Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:24:00 +0300
Subject: [PATCH 05/44] Add test_same_url

---
 tests/test_contracts.py | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b07cbee1e..e4f1a777b 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,8 +1,10 @@
 from unittest import TextTestResult
 
+from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
 
+from scrapy.crawler import CrawlerRunner
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
@@ -101,6 +103,29 @@ class TestSpider(Spider):
         pass
 
 
+class TestSameUrlSpider(Spider):
+
+    name = 'test_same_url'
+
+    def __init__(self, *args, **kwargs):
+        super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+        self.visited = 0
+
+    def parse_first(self, response):
+        """first callback
+        @url http://scrapy.org
+        """
+        self.visited += 1
+        return TestItem()
+
+    def parse_second(self, response):
+        """second callback
+        @url http://scrapy.org
+        """
+        self.visited += 1
+        return TestItem()
+
+
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
@@ -177,14 +202,12 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_succeed()
 
         # scrapes_item_fail
-        request = self.conman.from_method(spider.scrapes_item_fail,
-                self.results)
+        request = self.conman.from_method(spider.scrapes_item_fail, self.results)
         request.callback(response)
         self.should_fail()
 
         # scrapes_dict_item_fail
-        request = self.conman.from_method(spider.scrapes_dict_item_fail,
-                self.results)
+        request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
         request.callback(response)
         self.should_fail()
 
@@ -202,3 +225,12 @@ class ContractsManagerTest(unittest.TestCase):
 
         self.assertFalse(self.results.failures)
         self.assertTrue(self.results.errors)
+
+    @defer.inlineCallbacks
+    def test_same_url(self):
+        TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
+
+        crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+        yield crawler.crawl()
+
+        self.assertEqual(crawler.spider.visited, 2)

From 2cb4decb6ad52548b3441c877235c540cef12082 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:36:10 +0300
Subject: [PATCH 06/44] Move TestSameUrlSpider to test method

---
 tests/test_contracts.py | 45 ++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index e4f1a777b..430d89253 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -103,29 +103,6 @@ class TestSpider(Spider):
         pass
 
 
-class TestSameUrlSpider(Spider):
-
-    name = 'test_same_url'
-
-    def __init__(self, *args, **kwargs):
-        super(TestSameUrlSpider, self).__init__(*args, **kwargs)
-        self.visited = 0
-
-    def parse_first(self, response):
-        """first callback
-        @url http://scrapy.org
-        """
-        self.visited += 1
-        return TestItem()
-
-    def parse_second(self, response):
-        """second callback
-        @url http://scrapy.org
-        """
-        self.visited += 1
-        return TestItem()
-
-
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
@@ -228,6 +205,28 @@ class ContractsManagerTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_same_url(self):
+
+        class TestSameUrlSpider(Spider):
+            name = 'test_same_url'
+
+            def __init__(self, *args, **kwargs):
+                super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+                self.visited = 0
+
+            def parse_first(self, response):
+                """first callback
+                @url http://scrapy.org
+                """
+                self.visited += 1
+                return TestItem()
+
+            def parse_second(self, response):
+                """second callback
+                @url http://scrapy.org
+                """
+                self.visited += 1
+                return TestItem()
+
         TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
 
         crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)

From ddd69f4c10578975658ef6ac450cbe1bed85df80 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:39:43 +0300
Subject: [PATCH 07/44] Use MockServer in test_same_url

---
 tests/test_contracts.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 430d89253..223a926f1 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -15,6 +15,7 @@ from scrapy.contracts.default import (
     ReturnsContract,
     ScrapesContract,
 )
+from tests.mockserver import MockServer
 
 
 class TestItem(Item):
@@ -213,6 +214,9 @@ class ContractsManagerTest(unittest.TestCase):
                 super(TestSameUrlSpider, self).__init__(*args, **kwargs)
                 self.visited = 0
 
+            def start_requests(s):
+                return self.conman.from_spider(s, self.results)
+
             def parse_first(self, response):
                 """first callback
                 @url http://scrapy.org
@@ -227,9 +231,8 @@ class ContractsManagerTest(unittest.TestCase):
                 self.visited += 1
                 return TestItem()
 
-        TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
-
         crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
-        yield crawler.crawl()
+        with MockServer() as mockserver:
+            yield crawler.crawl(mockserver=mockserver)
 
         self.assertEqual(crawler.spider.visited, 2)

From 2b212d426668c02704bc23c4e988fa108e0738ef Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:06 +0000
Subject: [PATCH 08/44] ignore cache for pytests

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 406146e5f..ff6e2ea65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ dist
 .idea
 htmlcov/
 .coverage
+.pytest_cache/
 .coverage.*
 .cache/
 

From d95762db7c9abe59d8edfc8c17397a5a5bbfc661 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:24 +0000
Subject: [PATCH 09/44] new signal

---
 scrapy/signals.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/signals.py b/scrapy/signals.py
index e36c27203..c0e4bb74e 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -13,6 +13,7 @@ spider_closed = object()
 spider_error = object()
 request_scheduled = object()
 request_dropped = object()
+request_reached_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()

From 597b8a97ad468123432dba3d6f2c3dd943f4aa36 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:42 +0000
Subject: [PATCH 10/44] documentation for new signal

---
 docs/topics/signals.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index d40c0e1df..cf7b8db2f 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -279,6 +279,22 @@ request_dropped
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_reached_downloader
+---------------------------
+
+.. signal:: request_reached_downloader
+.. function:: request_reached_downloader(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request`, reached downloader.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 

From afb1458bd3b75fad78f053ac062910052971cdfc Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:54 +0000
Subject: [PATCH 11/44] tests for new signal

---
 tests/test_engine.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 719c0c60c..856465161 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -103,6 +103,7 @@ class CrawlerRun(object):
         self.respplug = []
         self.reqplug = []
         self.reqdropped = []
+        self.reqreached = []
         self.itemerror = []
         self.itemresp = []
         self.signals_catched = {}
@@ -124,6 +125,7 @@ class CrawlerRun(object):
         self.crawler.signals.connect(self.item_error, signals.item_error)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
+        self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
         self.crawler.crawl(start_urls=start_urls)
         self.spider = self.crawler.spider
@@ -155,6 +157,9 @@ class CrawlerRun(object):
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
+    def request_reached(self, request, spider):
+        self.reqreached.append((request, spider))
+
     def request_dropped(self, request, spider):
         self.reqdropped.append((request, spider))
 
@@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase):
         responses_count = len(self.run.respplug)
         self.assertEqual(scheduled_requests_count,
                          dropped_requests_count + responses_count)
+        self.assertEqual(len(self.run.reqreached),
+                         responses_count)
 
     def _assert_dropped_requests(self):
         self.assertEqual(len(self.run.reqdropped), 1)
@@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase):
     def _assert_downloaded_responses(self):
         # response tests
         self.assertEqual(8, len(self.run.respplug))
+        self.assertEqual(8, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':

From 561ad3b63c539b560bbaed1df37d0b60043ab3b3 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:40:24 +0000
Subject: [PATCH 12/44] emit new signal

---
 scrapy/core/downloader/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index d835e65f7..59c3ad074 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -129,6 +129,9 @@ class Downloader(object):
             return response
 
         slot.active.add(request)
+        self.signals.send_catch_log(signal=signals.request_reached_downloader,
+                                    request=request,
+                                    spider=spider)
         deferred = defer.Deferred().addBoth(_deactivate)
         slot.queue.append((request, deferred))
         self._process_queue(spider, slot)

From e2de0a7203b99d8fc71e62539fdd75dad439982a Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 18 Aug 2018 15:24:30 +0300
Subject: [PATCH 13/44] Use except Exception

---
 scrapy/contracts/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index c62df5ab0..18c59ff22 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -44,7 +44,7 @@ class ContractsManager(object):
             bound_method = spider.__getattribute__(method)
             try:
                 requests.append(self.from_method(bound_method, results))
-            except:
+            except Exception:
                 case = _create_testcase(bound_method, 'contract')
                 results.addError(case, sys.exc_info())
 

From 0467737cf0bc4c5603bd63a87e220482eb724600 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 18 Aug 2018 15:42:21 +0300
Subject: [PATCH 14/44] Fix mockserver usage

---
 tests/test_contracts.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c5820f256..0a216b745 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -222,22 +222,20 @@ class ContractsManagerTest(unittest.TestCase):
                 return self.conman.from_spider(s, self.results)
 
             def parse_first(self, response):
-                """first callback
-                @url {}
-                """.format(self.mockserver.url('/status?n=200'))
                 self.visited += 1
                 return TestItem()
 
             def parse_second(self, response):
-                """second callback
-                @url {}
-                """.format(self.mockserver.url('/status?n=200'))
                 self.visited += 1
                 return TestItem()
 
-        crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
         with MockServer() as mockserver:
-            yield crawler.crawl(mockserver=mockserver)
+            mock_endpoint = mockserver.url('/status?n=200')
+            TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+            TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+
+            crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+            yield crawler.crawl()
 
         self.assertEqual(crawler.spider.visited, 2)
 

From 57824600a8295f0d8537de2a8d1c3eb9977ed36d Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sun, 19 Aug 2018 16:56:41 +0300
Subject: [PATCH 15/44] Use six.get_unbound_function in test_same_url

---
 tests/test_contracts.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 0a216b745..b4209e1f6 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,5 +1,6 @@
 from unittest import TextTestResult
 
+from six import get_unbound_function
 from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
@@ -230,9 +231,10 @@ class ContractsManagerTest(unittest.TestCase):
                 return TestItem()
 
         with MockServer() as mockserver:
-            mock_endpoint = mockserver.url('/status?n=200')
-            TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint)
-            TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+            contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
+
+            get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
+            get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
 
             crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
             yield crawler.crawl()

From c02cfa574cc47d6b086cc66025ddef9f3174ac02 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 29 Aug 2018 11:21:55 +0000
Subject: [PATCH 16/44] remove comma

---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index cf7b8db2f..ff07b9d55 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -285,7 +285,7 @@ request_reached_downloader
 .. signal:: request_reached_downloader
 .. function:: request_reached_downloader(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request`, reached downloader.
+    Sent when a :class:`~scrapy.http.Request` reached downloader.
 
     The signal does not support returning deferreds from their handlers.
 

From 8dbbbd13950dcb21dda759b073c64ffdca85c2d6 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Mon, 3 Sep 2018 20:07:37 +0300
Subject: [PATCH 17/44] Use request_cls attribute in contract definition

---
 docs/topics/contracts.rst    |  5 +++--
 scrapy/contracts/__init__.py | 15 +++++++++------
 tests/test_contracts.py      |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ada6fd227..70f20d4ed 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -86,8 +86,9 @@ override three methods:
     .. method:: Contract.adjust_request_args(args)
 
         This receives a ``dict`` as an argument containing default arguments
-        for request object. :class:`~scrapy.http.Request` is used
-        if ``request_cls`` is not set on ``args``.
+        for request object. :class:`~scrapy.http.Request` is used by default,
+        but this can be changed with the ``request_cls`` attribute.
+        If multiple contracts in chain have this attribute defined, the last one is used.
 
         Must return the same or a modified version of it.
 
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 801c18e73..851a26a8e 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -4,7 +4,6 @@ from functools import wraps
 from inspect import getmembers
 from unittest import TestCase
 
-from scrapy import FormRequest
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
@@ -50,14 +49,17 @@ class ContractsManager(object):
     def from_method(self, method, results):
         contracts = self.extract_contracts(method)
         if contracts:
-            # prepare request arguments
-            kwargs = {'callback': method}
+            request_cls = Request
+            for contract in contracts:
+                if contract.request_cls is not None:
+                    request_cls = contract.request_cls
+
+            # calculate request args
+            args, kwargs = get_spec(request_cls.__init__)
+            kwargs['callback'] = method
             for contract in contracts:
                 kwargs = contract.adjust_request_args(kwargs)
 
-            request_cls = kwargs.pop('request_cls', Request)
-
-            args, _ = get_spec(request_cls.__init__)
             args.remove('self')
 
             # check if all positional arguments are defined in kwargs
@@ -98,6 +100,7 @@ class ContractsManager(object):
 
 class Contract(object):
     """ Abstract class for contracts """
+    request_cls = None
 
     def __init__(self, method, *args):
         self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c35b068a4..fc5c94771 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -27,9 +27,9 @@ class ResponseMock(object):
 
 class CustomFormContract(Contract):
     name = 'custom_form'
+    request_cls = FormRequest
 
     def adjust_request_args(self, args):
-        args['request_cls'] = FormRequest
         args['formdata'] = {'name': 'scrapy'}
         return args
 

From 25ac4691b414e7c18a4e9dec3bb6a85563d2488d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 9 Aug 2018 03:32:46 +0500
Subject: [PATCH 18/44] require parsel 1.5+

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 setup.py             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 03b33d02d..0771aae3a 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -6,5 +6,5 @@ queuelib
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index b38c4cc09..5a5d4c95a 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -6,5 +6,5 @@ queuelib>=1.1.1
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/setup.py b/setup.py
index c37919cda..8c47f67ce 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=1.4',
+        'parsel>=1.5',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 0ccead6681c9e2bf1902cbc1d4bde543be7d73e7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:36 +0500
Subject: [PATCH 19/44] DOC more Python 3 in examples

---
 docs/topics/commands.rst  |  6 ++---
 docs/topics/items.rst     |  6 ++---
 docs/topics/jobs.rst      |  4 ++--
 docs/topics/loaders.rst   | 14 ++++++------
 docs/topics/selectors.rst | 48 +++++++++++++++++++--------------------
 docs/topics/settings.rst  |  2 +-
 6 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3088017cb..ef9c45196 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -458,9 +458,9 @@ Usage example::
 
     >>> STATUS DEPTH LEVEL 1 <<<
     # Scraped Items  ------------------------------------------------------------
-    [{'name': u'Example item',
-     'category': u'Furniture',
-     'length': u'12 cm'}]
+    [{'name': 'Example item',
+     'category': 'Furniture',
+     'length': '12 cm'}]
 
     # Requests  -----------------------------------------------------------------
     []
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4423bbda2..ae44aecd3 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -86,7 +86,7 @@ Creating items
 ::
 
     >>> product = Product(name='Desktop PC', price=1000)
-    >>> print product
+    >>> print(product)
     Product(name='Desktop PC', price=1000)
 
 Getting field values
@@ -161,11 +161,11 @@ Other common tasks
 Copying items::
 
     >>> product2 = Product(product)
-    >>> print product2
+    >>> print(product2)
     Product(name='Desktop PC', price=1000)
 
     >>> product3 = product2.copy()
-    >>> print product3
+    >>> print(product3)
     Product(name='Desktop PC', price=1000)
 
 Creating dicts from items::
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 06c7fff3d..8e1574376 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -84,7 +84,7 @@ So, for example, this won't work::
         return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 But this will::
 
@@ -94,7 +94,7 @@ But this will::
 
     def other_callback(self, response):
         somearg = response.meta['somearg']
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a895b535c..f3b6aa4a1 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -678,10 +678,10 @@ Here is a list of all built-in processors:
         >>> from scrapy.loader.processors import Join
         >>> proc = Join()
         >>> proc(['one', 'two', 'three'])
-        u'one two three'
+        'one two three'
         >>> proc = Join('<br>')
         >>> proc(['one', 'two', 'three'])
-        u'one<br>two<br>three'
+        'one<br>two<br>three'
 
 .. class:: Compose(\*functions, \**default_loader_context)
 
@@ -744,9 +744,9 @@ Here is a list of all built-in processors:
         ...     return None if x == 'world' else x
         ...
         >>> from scrapy.loader.processors import MapCompose
-        >>> proc = MapCompose(filter_world, unicode.upper)
-        >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
-        [u'HELLO, u'THIS', u'IS', u'SCRAPY']
+        >>> proc = MapCompose(filter_world, str.upper)
+        >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+        ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
     constructor keyword arguments are used as default context values. See
@@ -772,7 +772,7 @@ Here is a list of all built-in processors:
         >>> import json
         >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
         >>> proc_single_json_str('{"foo": "bar"}')
-        u'bar'
+        'bar'
         >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
         >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-        [u'bar']
+        ['bar']
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ac40c3cc..25c1f0aab 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -235,17 +235,17 @@ Here's an example used to extract image names from the :ref:`HTML code
 <topics-selectors-htmlcode>` above::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
-    [u'My image 1',
-     u'My image 2',
-     u'My image 3',
-     u'My image 4',
-     u'My image 5']
+    ['My image 1',
+     'My image 2',
+     'My image 3',
+     'My image 4',
+     'My image 5']
 
 There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
 named ``.re_first()``. Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
-    u'My image 1'
+    'My image 1'
 
 .. _topics-selectors-relative-xpaths:
 
@@ -431,26 +431,26 @@ with groups of itemscopes and corresponding itemprops::
     ...     print "    properties:", props.extract()
     ...     print
 
-    current scope: [u'http://schema.org/Product']
-        properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+    current scope: ['http://schema.org/Product']
+        properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
 
-    current scope: [u'http://schema.org/AggregateRating']
-        properties: [u'ratingValue', u'reviewCount']
+    current scope: ['http://schema.org/AggregateRating']
+        properties: ['ratingValue', 'reviewCount']
 
-    current scope: [u'http://schema.org/Offer']
-        properties: [u'price', u'availability']
+    current scope: ['http://schema.org/Offer']
+        properties: ['price', 'availability']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
     >>>
 
@@ -543,22 +543,22 @@ Example::
 This gets all first ``<li>``  elements under whatever it is its parent::
 
     >>> xp("//li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element in the whole document::
 
     >>> xp("(//li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 This gets all first ``<li>``  elements under an ``<ul>``  parent::
 
     >>> xp("//ul/li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
 
     >>> xp("(//ul/li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 When querying by class, consider using CSS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1f1217770..47b6cf13d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -871,7 +871,7 @@ LOG_STDOUT
 Default: ``False``
 
 If ``True``, all standard output (and error) of your process will be redirected
-to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
+to the log. For example if you ``print('hello')`` it will appear in the Scrapy
 log.
 
 .. setting:: LOG_SHORT_NAMES

From 395d9d033a39ae9e82337ce50cffabd1ecedb702 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:57 +0500
Subject: [PATCH 20/44] add pytest temp files to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ff6e2ea65..7392ed31e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
+.pytest_cache/
 
 # Windows
 Thumbs.db

From ca27010d4f8c35d1c98259e56be797a18b044304 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:22:56 +0500
Subject: [PATCH 21/44] DOC .extract_first() -> .get()

---
 docs/intro/overview.rst |  6 ++---
 docs/intro/tutorial.rst | 49 +++++++++++++++++++++--------------------
 docs/topics/shell.rst   |  2 +-
 docs/topics/spiders.rst | 10 ++++-----
 4 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6f1c2c43f..9d7c94d39 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.xpath('span/small/text()').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.xpath('span/small/text()').get(),
                 }
 
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr("href")').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fa6dc274d..a24cf0f5b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -269,23 +269,24 @@ The other thing is that the result of calling ``.extract()`` is a list, because
 we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
 you know you just want the first result, as in this case, you can do::
 
-    >>> response.css('title::text').extract_first()
+    >>> response.css('title::text').get()
     'Quotes to Scrape'
 
 As an alternative, you could've written::
 
-    >>> response.css('title::text')[0].extract()
+    >>> response.css('title::text')[0].get()
     'Quotes to Scrape'
 
-However, using ``.extract_first()`` avoids an ``IndexError`` and returns
-``None`` when it doesn't find any element matching the selection.
+However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
+instance avoids an ``IndexError`` and returns ``None`` when it doesn't
+find any element matching the selection.
 
 There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
 Besides the :meth:`~scrapy.selector.Selector.extract` and
-:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
+:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
 expressions`::
 
@@ -314,7 +315,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
 
     >>> response.xpath('//title')
     [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
@@ -383,10 +384,10 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text::text").extract_first()
+    >>> title = quote.css("span.text::text").get()
     >>> title
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author::text").extract_first()
+    >>> author = quote.css("small.author::text").get()
     >>> author
     'Albert Einstein'
 
@@ -401,8 +402,8 @@ Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::
 
     >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text::text").extract_first()
-    ...     author = quote.css("small.author::text").extract_first()
+    ...     text = quote.css("span.text::text").get()
+    ...     author = quote.css("small.author::text").get()
     ...     tags = quote.css("div.tags a.tag::text").extract()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
@@ -434,8 +435,8 @@ in the callback, as you can see below::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
@@ -508,14 +509,14 @@ markup:
 
 We can try extracting it in the shell::
 
-    >>> response.css('li.next a').extract_first()
+    >>> response.css('li.next a').get()
     '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
 Scrapy supports a CSS extension that let's you select the attribute contents,
 like this::
 
-    >>> response.css('li.next a::attr(href)').extract_first()
+    >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
@@ -533,12 +534,12 @@ page, extracting data from it::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
@@ -584,12 +585,12 @@ As a shortcut for creating Request objects you can use
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('span small::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, callback=self.parse)
 
@@ -641,7 +642,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).extract_first().strip()
+                return response.css(query).get().strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
@@ -710,11 +711,11 @@ with a specific tag, building the URL based on the argument::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 11ab199f2..9de6abef7 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -179,7 +179,7 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
     >>> fetch("https://reddit.com")
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 697732b47..4505b7315 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules::
             self.logger.info('Hi, this is an item page! %s', response.url)
             item = scrapy.Item()
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
-            item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
-            item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
+            item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
+            item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             return item
 
 
@@ -548,9 +548,9 @@ These spiders are pretty easy to use, let's have a look at one example::
             self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
 
             item = TestItem()
-            item['id'] = node.xpath('@id').extract()
-            item['name'] = node.xpath('name').extract()
-            item['description'] = node.xpath('description').extract()
+            item['id'] = node.xpath('@id').get()
+            item['name'] = node.xpath('name').get()
+            item['description'] = node.xpath('description').get()
             return item
 
 Basically what we did up there was to create a spider that downloads a feed from

From d32c4deaa99bf0155542d1b508b5535065b5d7a9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:23:31 +0500
Subject: [PATCH 22/44] DOC update Scrapy selectors tutorial to match parsel's
 tutorial better

---
 docs/topics/selectors.rst | 471 +++++++++++++++++++++++++++-----------
 1 file changed, 340 insertions(+), 131 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 25c1f0aab..00158ecf1 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -6,7 +6,7 @@ Selectors
 
 When you're scraping web pages, the most common task you need to perform is
 to extract data from the HTML source. There are several libraries available to
-achieve this:
+achieve this, such as:
 
  * `BeautifulSoup`_ is a very popular web scraping library among Python
    programmers which constructs a Python object based on the structure of the
@@ -25,8 +25,9 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are built over the `lxml`_ library, which means they're very
-similar in speed and parsing accuracy.
+Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
+under the hood. It means Scrapy selectors are very similar in speed and
+parsing accuracy to lxml.
 
 This page explains how selectors work and describes their API which is very
 small and simple, unlike the `lxml`_ API which is much bigger because the
@@ -42,7 +43,7 @@ For a complete reference of the selectors API see
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
 .. _XPath: https://www.w3.org/TR/xpath
 .. _CSS: https://www.w3.org/TR/selectors
-
+.. _parsel: https://parsel.readthedocs.io/
 
 Using selectors
 ===============
@@ -63,21 +64,32 @@ input type::
 Constructing from text::
 
     >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
 
 Constructing from response::
 
     >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
 
 For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible::
+it's totally OK to use this shortcut when possible. By using it you can
+ensure the response body is parsed only once::
 
-    >>> response.selector.xpath('//span/text()').extract()
-    [u'good']
+    >>> response.selector.xpath('//span/text()').get()
+    'good'
 
+Querying responses using XPath and CSS is so common that responses include two
+more shortcuts: ``response.xpath()`` and ``response.css()``::
+
+    >>> response.xpath('//span/text()').get()
+    'good'
+    >>> response.css('span::text').get()
+    'good'
+
+Usually there is no need to construct Scrapy selectors manually because of
+these shortcuts.
 
 Using selectors
 ---------------
@@ -90,7 +102,7 @@ documentation server:
 
 .. _topics-selectors-htmlcode:
 
-Here's its HTML code:
+For the sake of completeness, here's its full HTML code:
 
 .. literalinclude:: ../_static/selectors-sample1.html
    :language: html
@@ -111,90 +123,179 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
 So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
 page, let's construct an XPath for selecting the text inside the title tag::
 
-    >>> response.selector.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-
-Querying responses using XPath and CSS is so common that responses include two
-convenience shortcuts: ``response.xpath()`` and ``response.css()``::
-
     >>> response.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-    >>> response.css('title::text')
-    [<Selector (text) xpath=//title/text()>]
+    [<Selector xpath='//title/text()' data='Example website'>]
+
+To actually extract the textual data, you must call the selector ``.get()``
+or ``.getall()`` methods, as follows::
+
+    >>> response.xpath('//title/text()').getall()
+    ['Example website']
+    >>> response.xpath('//title/text()').get()
+    'Example website'
+
+``.get()`` always returns a single result; if there are several matches,
+content of a first match is returned; if there are no matches, None
+is returned. ``.getall()`` returns a list with all results.
+
+Notice that CSS selectors can select text or attribute nodes using CSS3
+pseudo-elements::
+
+    >>> selector.css('title::text').get()
+    'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
 selectors. This API can be used for quickly selecting nested data::
 
-    >>> response.css('img').xpath('@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('img').xpath('@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
 
-To actually extract the textual data, you must call the selector ``.extract()``
-method, as follows::
+If you want to extract only the first matched element, you can call the
+selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
+previous Scrapy versions)::
 
-    >>> response.xpath('//title/text()').extract()
-    [u'Example website']
+    >>> response.xpath('//div[@id="images"]/a/text()').get()
+    'Name: My image 1 '
 
-If you want to extract only first matched element, you can call the selector ``.extract_first()``
+It returns ``None`` if no element was found::
 
-    >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
-    u'Name: My image 1 '
-
-It returns ``None`` if no element was found:
-
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
     True
 
-A default return value can be provided as an argument, to be used instead of ``None``:
+A default return value can be provided as an argument, to be used instead
+of ``None``:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+    >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
     'not-found'
 
-Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
 
-    >>> response.css('title::text').extract()
-    [u'Example website']
+    >>> [img.attrib['src'] for img in response.css('img')]
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+As a shortcut, ``.attrib`` is also available on SelectorList directly;
+it returns attributes for the first matching element::
+
+    >>> response.css('img').attrib['src']
+    'image1_thumb.jpg'
+
+This is most useful when only a single result is expected, e.g. when selecting
+by id, or selecting unique elements on a web page::
+
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
 Now we're going to get the base URL and some image links::
 
-    >>> response.xpath('//base/@href').extract()
-    [u'http://example.com/']
+    >>> response.xpath('//base/@href').get()
+    'http://example.com/'
 
-    >>> response.css('base::attr(href)').extract()
-    [u'http://example.com/']
+    >>> response.css('base::attr(href)').get()
+    'http://example.com/'
 
-    >>> response.xpath('//a[contains(@href, "image")]/@href').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
-    >>> response.css('a[href*=image]::attr(href)').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('a[href*=image]::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.css('a[href*=image] img::attr(src)').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+    >>> response.css('a[href*=image] img::attr(src)').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+.. _topics-selectors-css-extensions:
+
+Extensions to CSS Selectors
+---------------------------
+
+Per W3C standards, `CSS selectors`_ do not support selecting text nodes
+or attribute values.
+But selecting these is so essential in a web scraping context
+that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
+
+* to select text nodes, use ``::text``
+* to select attribute values, use ``::attr(name)`` where *name* is the
+  name of the attribute that you want the value of
+
+.. warning::
+    These pseudo-elements are Scrapy-/Parsel-specific.
+    They will most probably not work with other libraries like
+   `lxml`_ or `PyQuery`_.
+
+.. _PyQuery: https://pypi.python.org/pypi/pyquery
+
+Examples:
+
+* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
+
+    >>> response.css('title::text').get()
+    'Example website'
+
+* ``*::text`` selects all descendant text nodes of the current selector context::
+
+    >>> response.css('#images *::text').getall()
+    ['\n   ',
+     'Name: My image 1 ',
+     '\n   ',
+     'Name: My image 2 ',
+     '\n   ',
+     'Name: My image 3 ',
+     '\n   ',
+     'Name: My image 4 ',
+     '\n   ',
+     'Name: My image 5 ',
+     '\n  ']
+
+* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
+
+.. note::
+    You cannot chain these pseudo-elements. But in practice it would not
+    make much sense: text nodes do not have attributes, and attribute values
+    are string values already and do not have children nodes.
+
+.. note::
+    See also: :ref:`selecting-attributes`.
+
+
+.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors:
 
@@ -206,22 +307,65 @@ of the same type, so you can call the selection methods for those selectors
 too. Here's an example::
 
     >>> links = response.xpath('//a[contains(@href, "image")]')
-    >>> links.extract()
-    [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
-     u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
-     u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
-     u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
-     u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+    >>> links.getall()
+    ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+     '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+     '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+     '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+     '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
 
     >>> for index, link in enumerate(links):
-    ...     args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract())
-    ...     print 'Link number %d points to url %s and image %s' % args
+    ...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+    ...     print('Link number %d points to url %r and image %r' % args)
 
-    Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
-    Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg']
-    Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg']
-    Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg']
-    Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg']
+    Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+    Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+    Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+    Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+    Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+
+.. _selecting-attributes:
+
+Selecting element attributes
+----------------------------
+
+There are several ways to get a value of an attribute. First, one can use
+XPath syntax::
+
+    >>> response.xpath("//a/@href").getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+XPath syntax has a few advantages: it is a standard XPath feature, and
+``@attributes`` can be used in other parts of an XPath expression - e.g.
+it is possible to filter by attribute value.
+
+Scrapy also provides an extension to CSS selectors (``::attr(...)``)
+which allows to get attribute values::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+In addition to that, there is a ``.attrib`` property of Selector.
+You can use it if you prefer to lookup attributes in Python
+code, without using XPaths or CSS extensions::
+
+    >>> [a.attrib['href'] for a in response.css('a')]
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+This property is also available on SelectorList; it returns a dictionary
+with attributes of a first matching element. It is convenient to use when
+a selector is expected to give a single result (e.g. when selecting by element
+ID, or when selecting an unique element on a page)::
+
+    >>> response.css('base').attrib
+    {'href': 'http://example.com/'}
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
+
+``.attrib`` property of an empty SelectorList is empty::
+
+    >>> response.css('foo').attrib
+    {}
 
 Using selectors with regular expressions
 ----------------------------------------
@@ -241,8 +385,9 @@ Here's an example used to extract image names from the :ref:`HTML code
      'My image 4',
      'My image 5']
 
-There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
-named ``.re_first()``. Use it to extract just the first matching string::
+There's an additional helper reciprocating ``.get()`` (and its
+alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
+Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
@@ -266,17 +411,17 @@ it actually extracts all ``<p>`` elements from the document, not only those
 inside ``<div>`` elements::
 
     >>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
-    ...     print p.extract()
+    ...     print(p.get())
 
 This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
 
     >>> for p in divs.xpath('.//p'):  # extracts all <p> inside
-    ...     print p.extract()
+    ...     print(p.get())
 
 Another common case would be to extract all direct ``<p>`` children::
 
     >>> for p in divs.xpath('p'):
-    ...     print p.extract()
+    ...     print(p.get())
 
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
@@ -298,14 +443,14 @@ Here's an example to match an element based on its "id" attribute value,
 without hard-coding it (that was shown previously)::
 
     >>> # `$val` used in the expression, a `val` argument needs to be passed
-    >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
-    u'Name: My image 1 '
+    >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+    'Name: My image 1 '
 
 Here's another example, to find the "id" attribute of a ``<div>`` tag containing
 five ``<a>`` children (here we pass the value ``5`` as an integer)::
 
-    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
-    u'images'
+    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+    'images'
 
 All variable references must have a binding value when calling ``.xpath()``
 (otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -314,13 +459,12 @@ This is done by passing as many named arguments as necessary.
 `parsel`_, the library powering Scrapy selectors, has more details and examples
 on `XPath variables`_.
 
-.. _parsel: https://parsel.readthedocs.io/
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
 Using EXSLT extensions
 ----------------------
 
-Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions
 and come with these pre-registered namespaces to use in XPath expressions:
 
 
@@ -340,7 +484,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit::
 
     >>> from scrapy import Selector
-    >>> doc = """
+    >>> doc = u"""
     ... <div>
     ...     <ul>
     ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -352,10 +496,10 @@ Example selecting links in list item with a "class" attribute ending with a digi
     ... </div>
     ... """
     >>> sel = Selector(text=doc, type="html")
-    >>> sel.xpath('//li//@href').extract()
-    [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
-    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
-    [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+    >>> sel.xpath('//li//@href').getall()
+    ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+    ['link1.html', 'link2.html', 'link4.html', 'link5.html']
     >>>
 
 .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
@@ -372,7 +516,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from http://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = """
+    >>> doc = u"""
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@@ -424,12 +568,12 @@ with groups of itemscopes and corresponding itemprops::
     ... """
     >>> sel = Selector(text=doc, type="html")
     >>> for scope in sel.xpath('//div[@itemscope]'):
-    ...     print "current scope:", scope.xpath('@itemtype').extract()
+    ...     print("current scope:", scope.xpath('@itemtype').getall())
     ...     props = scope.xpath('''
     ...                 set:difference(./descendant::*/@itemprop,
     ...                                .//*[@itemscope]/*/@itemprop)''')
-    ...     print "    properties:", props.extract()
-    ...     print
+    ...     print("    properties: %s" % (props.getall()))
+    ...     print("")
 
     current scope: ['http://schema.org/Product']
         properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
@@ -493,27 +637,27 @@ Example::
 
 Converting a *node-set* to string::
 
-    >>> sel.xpath('//a//text()').extract() # take a peek at the node-set
-    [u'Click here to go to the ', u'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string
-    [u'Click here to go to the ']
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
 
 A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
 
-    >>> sel.xpath("//a[1]").extract() # select the first node
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").extract() # convert it to string
-    [u'Click here to go to the Next Page']
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
 
 So, using the ``.//text()`` node-set won't select anything in this case::
 
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract()
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
     []
 
 But using the ``.`` to mean the node, works::
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
 .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
 
@@ -538,7 +682,7 @@ Example::
     ....:         <li>5</li>
     ....:         <li>6</li>
     ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).extract()
+    >>> xp = lambda x: sel.xpath(x).getall()
 
 This gets all first ``<li>``  elements under whatever it is its parent::
 
@@ -578,12 +722,59 @@ you can just select by class using CSS and then switch to XPath when needed::
 
     >>> from scrapy import Selector
     >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').extract()
-    [u'2014-07-23 19:00']
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
 
 This is cleaner than using the verbose XPath trick shown above. Just remember
 to use the ``.`` in the XPath expressions that will follow.
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. These methods
+are still supported by Scrapy, there are no plans to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
 
 .. _topics-selectors-ref:
 
@@ -718,10 +909,12 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples-html:
+
 Selector examples on HTML response
 ----------------------------------
 
-Here's a couple of :class:`Selector` examples to illustrate several concepts.
+Here are some :class:`Selector` examples to illustrate several concepts.
 In all cases, we assume there is already a :class:`Selector` instantiated with
 a :class:`~scrapy.http.HtmlResponse` object like this::
 
@@ -735,20 +928,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
    returning a list of unicode strings::
 
-      sel.xpath("//h1").extract()         # this includes the h1 tag
-      sel.xpath("//h1/text()").extract()  # this excludes the h1 tag
+      sel.xpath("//h1").getall()         # this includes the h1 tag
+      sel.xpath("//h1/text()").getall()  # this excludes the h1 tag
 
 3. Iterate over all ``<p>`` tags and print their class attribute::
 
       for node in sel.xpath("//p"):
-          print node.xpath("@class").extract()
+          print(node.attrib['class'])
+
+
+.. _selector-examples-xml:
 
 Selector examples on XML response
 ---------------------------------
 
-Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already a :class:`Selector` instantiated with an
-:class:`~scrapy.http.XmlResponse` object like this::
+Here are some examples to illustrate concepts for :class:`Selector` objects
+instantiated with an :class:`~scrapy.http.XmlResponse` object::
 
       sel = Selector(xml_response)
 
@@ -761,7 +956,7 @@ assume there is already a :class:`Selector` instantiated with an
    a namespace::
 
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
-      sel.xpath("//g:price").extract()
+      sel.xpath("//g:price").getall()
 
 .. _removing-namespaces:
 
@@ -781,6 +976,20 @@ First, we open the shell with the url we want to scrape::
 
     $ scrapy shell https://github.com/blog.atom
 
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
 .. highlight:: python
 
 Once in the shell we can try selecting all ``<link>`` objects and see that it
@@ -794,8 +1003,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default
@@ -803,8 +1012,8 @@ instead of having to call it manually, this is because of two reasons, which, in
 of relevance, are:
 
 1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform for all
-   documents crawled by Scrapy
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
 
 2. There could be some cases where using namespaces is actually required, in
    case some element names clash between namespaces. These cases are very rare

From 09fd6c2a816ebec175ff3ffd945ce3263c9f1570 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 03:59:58 +0500
Subject: [PATCH 23/44] DOC unlink Firefox & Firebug sections from the tutorial
 for now.

See https://github.com/scrapy/scrapy/issues/3373 and https://github.com/scrapy/scrapy/issues/3372 for motivation.
---
 docs/intro/tutorial.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a24cf0f5b..453f5114f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -299,7 +299,8 @@ expressions`::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
+You can use your browser developer tools to inspect the HTML and come up
+with a selector (see section about :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.

From 2c48d156db7d7846d80fa379dfb33552ac5b5a85 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:01:17 +0500
Subject: [PATCH 24/44] DOC cleanup references in tutorials:

* remove unused link
* fix ReST syntax
* fix a link to regular expression docs
---
 docs/intro/tutorial.rst   | 3 +--
 docs/topics/selectors.rst | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 453f5114f..92d1065af 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -288,7 +288,7 @@ to be scraped, you can at least get **some** data.
 Besides the :meth:`~scrapy.selector.Selector.extract` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
-expressions`::
+expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
     ['Quotes to Scrape']
@@ -740,4 +740,3 @@ modeling the scraped data. If you prefer to play with an example project, check
 the :ref:`intro-examples` section.
 
 .. _JSON: https://en.wikipedia.org/wiki/JSON
-.. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 00158ecf1..cf1b0ba22 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -251,7 +251,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
 .. warning::
     These pseudo-elements are Scrapy-/Parsel-specific.
     They will most probably not work with other libraries like
-   `lxml`_ or `PyQuery`_.
+    `lxml`_ or `PyQuery`_.
 
 .. _PyQuery: https://pypi.python.org/pypi/pyquery
 

From 53da56c8dcd6edba12284ed5d125ddd1dcf6f99c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:17:55 +0500
Subject: [PATCH 25/44] TST update tests to use get/getall/attrib instead of
 extract

---
 tests/test_command_shell.py   |  2 +-
 tests/test_http_response.py   | 34 +++++++++++-----------
 tests/test_loader.py          |  4 +--
 tests/test_pipeline_crawl.py  |  2 +-
 tests/test_selector.py        | 24 +++++++--------
 tests/test_spider.py          |  8 ++---
 tests/test_utils_iterators.py | 55 ++++++++++++++++++-----------------
 7 files changed, 66 insertions(+), 63 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 3e27d6abd..36baacfbd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_response_selector_html(self):
-        xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
+        xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
         _, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
         self.assertEqual(out.strip(), b'Works')
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 820758dc9..3b90e3dac 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//title/text()").extract(),
+            response.selector.xpath("//title/text()").getall(),
             [u'Some page']
         )
         self.assertEqual(
-            response.selector.css("title::text").extract(),
+            response.selector.css("title::text").getall(),
             [u'Some page']
         )
         self.assertEqual(
@@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//title/text()").extract(),
-            response.selector.xpath("//title/text()").extract(),
+            response.xpath("//title/text()").getall(),
+            response.selector.xpath("//title/text()").getall(),
         )
         self.assertEqual(
-            response.css("title::text").extract(),
-            response.selector.css("title::text").extract(),
+            response.css("title::text").getall(),
+            response.selector.css("title::text").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
-            response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
+            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
+            response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
         )
         self.assertEqual(
             response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1).extract(),
-            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
+                pclass="content", pcount=1).getall(),
+            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
     def test_urljoin_with_base_url(self):
@@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//elem/text()").extract(),
+            response.selector.xpath("//elem/text()").getall(),
             [u'value']
         )
 
@@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//elem/text()").extract(),
-            response.selector.xpath("//elem/text()").extract(),
+            response.xpath("//elem/text()").getall(),
+            response.selector.xpath("//elem/text()").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
         )
 
         response.selector.register_namespace('s2', 'http://scrapy.org')
         self.assertEqual(
-            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s2:elem/text()").extract(),
+            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s2:elem/text()").getall(),
         )
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 3b5714058..8b58e4dbd 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
@@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_css("header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5985a6f3e..fb72c9d6d 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
                     for href in response.xpath('''
                         //table[thead/tr/th="Filename"]
                             /tbody//a/@href
-                        ''').extract()],
+                        ''').getall()],
         }
         yield item
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 526660cc8..bc4baf7ea 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
         for x in xl:
             assert isinstance(x, Selector)
 
-        self.assertEqual(sel.xpath('//input').extract(),
-                         [x.extract() for x in sel.xpath('//input')])
+        self.assertEqual(sel.xpath('//input').getall(),
+                         [x.get() for x in sel.xpath('//input')])
 
-        self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
+        self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
                          [u'a'])
-        self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+        self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
                          [u'12.0'])
 
-        self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
+        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
                          [u'xpathrules'])
-        self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+        self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
     def test_root_base_url(self):
@@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
@@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
                           [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
-        r1 = TextResponse('http://www.example.com', \
-                          body=b'<html><p>an Jos\xe9 de</p><html>', \
+        r1 = TextResponse('http://www.example.com',
+                          body=b'<html><p>an Jos\xe9 de</p><html>',
                           encoding='utf-8')
-        Selector(r1).xpath('//text()').extract()
+        Selector(r1).xpath('//text()').getall()
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 929e0fea8..f26da2334 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
 
             def parse_node(self, response, selector):
                 yield {
-                    'loc': selector.xpath('a:loc/text()').extract(),
-                    'updated': selector.xpath('b:updated/text()').extract(),
-                    'other': selector.xpath('other/@value').extract(),
-                    'custom': selector.xpath('other/@b:custom').extract(),
+                    'loc': selector.xpath('a:loc/text()').getall(),
+                    'updated': selector.xpath('b:updated/text()').getall(),
+                    'other': selector.xpath('other/@value').getall(),
+                    'custom': selector.xpath('other/@b:custom').getall(),
                 }
 
         for iterator in ('iternodes', 'xml'):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f953076b8..00eb78068 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
         for x in self.xmliter(response, 'product'):
-            attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
+            attrs.append((
+                x.attrib['id'],
+                x.xpath("name/text()").extract(),
+                x.xpath("./type/text()").extract()))
 
         self.assertEqual(attrs,
-                         [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
+                         [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
 
     def test_xmliter_unusual_node(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').extract()
+        nodenames = [e.xpath('name()').getall()
                  for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
@@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
 
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
-                attrs.append((x.xpath('@id').extract(),
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+                attrs.append((x.attrib['id'],
+                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [([u'26'], [u'-'], [u'80']),
-                              ([u'21'], [u'Ab'], [u'76']),
-                              ([u'27'], [u'A'], [u'27'])])
+                             [(u'26', [u'-'], [u'80']),
+                              (u'21', [u'Ab'], [u'76']),
+                              (u'27', [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
 
-        self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
+        self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
@@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
 
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
-        self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
-        self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
-        self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
-        self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
-        self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
-        self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
-        self.assertEqual(node.xpath('image_link/text()').extract(), [])
-        self.assertEqual(node.xpath('id/text()').extract(), [])
-        self.assertEqual(node.xpath('price/text()').extract(), [])
+        self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
+        self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
+        self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+        self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
+        self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
+        self.assertEqual(node.xpath('image_link/text()').getall(), [])
+        self.assertEqual(node.xpath('id/text()').getall(), [])
+        self.assertEqual(node.xpath('price/text()').getall(), [])
 
     def test_xmliter_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            next(self.xmliter(response, 'item')).extract(),
+            next(self.xmliter(response, 'item')).get(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
@@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
 
         namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
         body = b"""\
@@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
         my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
 
         node = next(my_iter)
-        self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
-        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
-        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
+        self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
+        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
+        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
 
         my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
 
         node = next(my_iter)
-        self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+        self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
 
     def test_xmliter_objtype_exception(self):
         i = self.xmliter(42, 'product')

From 8c29be606c1be71059b454e41b87354e22569423 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:18:29 +0500
Subject: [PATCH 26/44] update spider templates to use .get

---
 scrapy/templates/spiders/crawl.tmpl   | 10 +++++-----
 scrapy/templates/spiders/xmlfeed.tmpl | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 802cb88a1..878425125 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -14,8 +14,8 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        i = {}
-        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
-        #i['name'] = response.xpath('//div[@id="name"]').extract()
-        #i['description'] = response.xpath('//div[@id="description"]').extract()
-        return i
+        item = {}
+        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+        #item['name'] = response.xpath('//div[@id="name"]').get()
+        #item['description'] = response.xpath('//div[@id="description"]').get()
+        return item
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 7c2ff8850..863c9772f 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
     itertag = 'item' # change it accordingly
 
     def parse_node(self, response, selector):
-        i = {}
-        #i['url'] = selector.select('url').extract()
-        #i['name'] = selector.select('name').extract()
-        #i['description'] = selector.select('description').extract()
-        return i
+        item = {}
+        #item['url'] = selector.select('url').get()
+        #item['name'] = selector.select('name').get()
+        #item['description'] = selector.select('description').get()
+        return item

From 460f0f045141f73dcc6d7a2309b7e2f0c2a492b4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:32 +0500
Subject: [PATCH 27/44] [backwards incompatible] switch ItemLoader from
 .extract to .getall.

This change is backwards incompatible if ItemLoader is used with a custom Selector
subclass which overrides .extract without overriding .getall.
---
 scrapy/loader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index e73413318..a7c75a46a 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -181,7 +181,7 @@ class ItemLoader(object):
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
+        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
 
     def add_css(self, field_name, css, *processors, **kw):
         values = self._get_cssvalues(css, **kw)
@@ -198,6 +198,6 @@ class ItemLoader(object):
     def _get_cssvalues(self, csss, **kw):
         self._check_selector_method()
         csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).extract() for css in csss)
+        return flatten(self.selector.css(css).getall() for css in csss)
 
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

From 12e42bbe06bea5a0f86e95630b184e86b41c95c7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:55 +0500
Subject: [PATCH 28/44] switch SgmlLinkExtractor to .getall

---
 scrapy/linkextractors/sgml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index f4ca4262a..5fa6b771c 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
             base_url = get_base_url(response)
             body = u''.join(f
                             for x in self.restrict_xpaths
-                            for f in response.xpath(x).extract()
+                            for f in response.xpath(x).getall()
                             ).encode(response.encoding, errors='xmlcharrefreplace')
         else:
             body = response.body

From afce9716fabff6169dc9cc146a2589ebf4c1ed13 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:25:43 +0500
Subject: [PATCH 29/44] DOC mention .attrib in the tutorial

---
 docs/intro/tutorial.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 92d1065af..07fb4807f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -520,6 +520,12 @@ like this::
     >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
+There is also an ``attrib`` property available
+(see :ref:`selecting-attributes` for more)::
+
+    >>> response.css('li.next a').attrib['href']
+    '/page/2'
+
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
 

From bdcc045f62a34d1655a4ead4e0b38d6fe97a08fc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:27:21 +0500
Subject: [PATCH 30/44] DOC switch from .extract to get/getall API in docs

Also, response.urljoin is added in a few places, for robustness.
---
 docs/intro/tutorial.rst | 22 +++++++++++-----------
 docs/topics/shell.rst   |  4 ++--
 docs/topics/spiders.rst | 14 +++++++-------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 07fb4807f..46e84b21c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -254,7 +254,7 @@ data.
 
 To extract the text from the title above, you can do::
 
-    >>> response.css('title::text').extract()
+    >>> response.css('title::text').getall()
     ['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
@@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside
 ``<title>`` element.  If we don't specify ``::text``, we'd get the full title
 element, including its tags::
 
-    >>> response.css('title').extract()
+    >>> response.css('title').getall()
     ['<title>Quotes to Scrape</title>']
 
-The other thing is that the result of calling ``.extract()`` is a list, because
-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
-you know you just want the first result, as in this case, you can do::
+The other thing is that the result of calling ``.getall()`` is a list: it is
+possible that a selector returns more than one result, so we extract them all.
+When you know you just want the first result, as in this case, you can do::
 
     >>> response.css('title::text').get()
     'Quotes to Scrape'
@@ -392,10 +392,10 @@ using the ``quote`` object we just created::
     >>> author
     'Albert Einstein'
 
-Given that the tags are a list of strings, we can use the ``.extract()`` method
+Given that the tags are a list of strings, we can use the ``.getall()`` method
 to get all of them::
 
-    >>> tags = quote.css("div.tags a.tag::text").extract()
+    >>> tags = quote.css("div.tags a.tag::text").getall()
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
@@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary::
     >>> for quote in response.css("div.quote"):
     ...     text = quote.css("span.text::text").get()
     ...     author = quote.css("small.author::text").get()
-    ...     tags = quote.css("div.tags a.tag::text").extract()
+    ...     tags = quote.css("div.tags a.tag::text").getall()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
     {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
@@ -438,7 +438,7 @@ in the callback, as you can see below::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
 If you run this spider, it will output the extracted data with the log::
@@ -543,7 +543,7 @@ page, extracting data from it::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
@@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can use
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('span small::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 9de6abef7..68a0b19b5 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -184,8 +184,8 @@ After that, we can start playing with the objects::
 
     >>> fetch("https://reddit.com")
 
-    >>> response.xpath('//title/text()').extract()
-    ['reddit: the front page of the internet']
+    >>> response.xpath('//title/text()').get()
+    'reddit: the front page of the internet'
 
     >>> request = request.replace(method="POST")
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 4505b7315..a08dc30f2 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::
         ]
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield {"title": h3}
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
 to give data more structure you can use :ref:`topics-items`::
@@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::
             yield scrapy.Request('http://www.example.com/3.html', self.parse)
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield MyItem(title=h3)
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 .. _spiderargs:
 
@@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example::
         itertag = 'item'
 
         def parse_node(self, response, node):
-            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
+            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
 
             item = TestItem()
             item['id'] = node.xpath('@id').get()

From 7fdfdb7fa244ed44274aaa9750efec1c333eedd1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 17:34:44 +0500
Subject: [PATCH 31/44] DOC reorganize selectors tutorial, port more topics
 from parsel docs, adjust wording in the introduction

---
 docs/topics/selectors.rst | 514 +++++++++++++++++++++-----------------
 1 file changed, 281 insertions(+), 233 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index cf1b0ba22..2db982b37 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -25,17 +25,14 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
-under the hood. It means Scrapy selectors are very similar in speed and
-parsing accuracy to lxml.
+.. note::
+    Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of
+    this wrapper is to provide better integration with Scrapy Response objects.
 
-This page explains how selectors work and describes their API which is very
-small and simple, unlike the `lxml`_ API which is much bigger because the
-`lxml`_ library can be used for many other tasks, besides selecting markup
-documents.
-
-For a complete reference of the selectors API see
-:ref:`Selector reference <topics-selectors-ref>`
+    `parsel`_ is a stand-alone web scraping library which can be used without
+    Scrapy. It uses `lxml`_ library under the hood, and implements an
+    easy API on top of lxml API. It means Scrapy selectors are very similar
+    in speed and parsing accuracy to lxml.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
@@ -73,9 +70,8 @@ Constructing from response::
     >>> Selector(response=response).xpath('//span/text()').get()
     'good'
 
-For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible. By using it you can
-ensure the response body is parsed only once::
+For convenience, response objects expose a selector on `.selector` attribute.
+By using it you can ensure the response body is parsed only once::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -88,8 +84,10 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
-Usually there is no need to construct Scrapy selectors manually because of
-these shortcuts.
+Usually there is no need to construct Scrapy selectors manually:
+``response`` object is available in Spider callbacks, so in most cases
+it is more convenient to use ``response.css()`` and ``response.xpath()``
+shortcuts.
 
 Using selectors
 ---------------
@@ -392,6 +390,71 @@ Use it to extract just the first matching string::
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts
+and tutorials are using them as well. These methods are still supported
+by Scrapy, there are **no plans** to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
+
+.. _topics-selectors-xpaths:
+
+Working with XPaths
+===================
+
+Here are some tips which may help you to use XPath with Scrapy selectors
+effectively. If you are not much familiar with XPath yet,
+you may want to take a look first at this `XPath tutorial`_.
+
+.. note::
+    Some of the tips are based on `this post from ScrapingHub's blog`_.
+
+.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+
+
 .. _topics-selectors-relative-xpaths:
 
 Working with relative XPaths
@@ -428,6 +491,114 @@ XPath specification.
 
 .. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
+When querying by class, consider using CSS
+------------------------------------------
+
+Because an element can contain multiple CSS classes, the XPath way to select elements
+by class is the rather verbose::
+
+    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
+
+If you use ``@class='someclass'`` you may end up missing elements that have
+other classes, and if you just use ``contains(@class, 'someclass')`` to make up
+for that you may end up with more elements that you want, if they have a different
+class name that shares the string ``someclass``.
+
+As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
+you can just select by class using CSS and then switch to XPath when needed::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
+
+This is cleaner than using the verbose XPath trick shown above. Just remember
+to use the ``.`` in the XPath expressions that will follow.
+
+Beware of the difference between //node[1] and (//node)[1]
+----------------------------------------------------------
+
+``//node[1]`` selects all the nodes occurring first under their respective parents.
+
+``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text="""
+    ....:     <ul class="list">
+    ....:         <li>1</li>
+    ....:         <li>2</li>
+    ....:         <li>3</li>
+    ....:     </ul>
+    ....:     <ul class="list">
+    ....:         <li>4</li>
+    ....:         <li>5</li>
+    ....:         <li>6</li>
+    ....:     </ul>""")
+    >>> xp = lambda x: sel.xpath(x).getall()
+
+This gets all first ``<li>``  elements under whatever it is its parent::
+
+    >>> xp("//li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element in the whole document::
+
+    >>> xp("(//li)[1]")
+    ['<li>1</li>']
+
+This gets all first ``<li>``  elements under an ``<ul>``  parent::
+
+    >>> xp("//ul/li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
+
+    >>> xp("(//ul/li)[1]")
+    ['<li>1</li>']
+
+Using text nodes in a condition
+-------------------------------
+
+When you need to use the text content as argument to an `XPath string function`_,
+avoid using ``.//text()`` and use just ``.`` instead.
+
+This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
+And when a node-set is converted to a string, which happens when it is passed as argument to
+a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+
+Converting a *node-set* to string::
+
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
+
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
+
+So, using the ``.//text()`` node-set won't select anything in this case::
+
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    []
+
+But using the ``.`` to mean the node, works::
+
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+
 .. _topics-selectors-xpath-variables:
 
 Variables in XPath expressions
@@ -461,6 +632,69 @@ on `XPath variables`_.
 
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
+
+.. _removing-namespaces:
+
+Removing namespaces
+-------------------
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names, to write more
+simple/convenient XPaths. You can use the
+:meth:`Selector.remove_namespaces` method for that.
+
+Let's show an example that illustrates this with GitHub blog atom feed.
+
+.. highlight:: sh
+
+First, we open the shell with the url we want to scrape::
+
+    $ scrapy shell https://github.com/blog.atom
+
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
+.. highlight:: python
+
+Once in the shell we can try selecting all ``<link>`` objects and see that it
+doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+
+    >>> response.xpath("//link")
+    []
+
+But once we call the :meth:`Selector.remove_namespaces` method, all
+nodes can be accessed directly by their names::
+
+    >>> response.selector.remove_namespaces()
+    >>> response.xpath("//link")
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     ...
+
+If you wonder why the namespace removal procedure isn't always called by default
+instead of having to call it manually, this is because of two reasons, which, in order
+of relevance, are:
+
+1. Removing namespaces requires to iterate and modify all nodes in the
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
+
+2. There could be some cases where using namespaces is actually required, in
+   case some element names clash between namespaces. These cases are very rare
+   though.
+
+
 Using EXSLT extensions
 ----------------------
 
@@ -606,174 +840,44 @@ inside another ``itemscope``.
 .. _regular expressions: http://exslt.org/regexp/index.html
 .. _set manipulation: http://exslt.org/set/index.html
 
+Other XPath extensions
+----------------------
 
-Some XPath tips
----------------
+Scrapy selectors also provide a sorely missed XPath extension function
+``has-class`` that returns ``True`` for nodes that have all of the specified
+HTML classes.
 
-Here are some tips that you may find useful when using XPath
-with Scrapy selectors, based on `this post from ScrapingHub's blog`_.
-If you are not much familiar with XPath yet,
-you may want to take a look first at this `XPath tutorial`_.
+.. highlight:: html
 
+For the following HTML::
 
-.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+    <p class="foo bar-baz">First</p>
+    <p class="foo">Second</p>
+    <p class="bar">Third</p>
+    <p>Fourth</p>
 
+.. highlight:: python
 
-Using text nodes in a condition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can use it like this::
 
-When you need to use the text content as argument to an `XPath string function`_,
-avoid using ``.//text()`` and use just ``.`` instead.
-
-This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
-And when a node-set is converted to a string, which happens when it is passed as argument to
-a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
-
-Converting a *node-set* to string::
-
-    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
-    ['Click here to go to the ', 'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
-    ['Click here to go to the ']
-
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
-
-    >>> sel.xpath("//a[1]").getall() # select the first node
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").getall() # convert it to string
-    ['Click here to go to the Next Page']
-
-So, using the ``.//text()`` node-set won't select anything in this case::
-
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    >>> response.xpath('//p[has-class("foo")]')
+    [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+     <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar-baz")]')
+    [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar")]')
     []
 
-But using the ``.`` to mean the node, works::
+So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
+``p.foo.bar-baz``.  Please note, that it is slower in most of the cases,
+because it's a pure-Python function that's invoked for every node in question
+whereas the CSS lookup is translated into XPath and thus runs more efficiently,
+so performance-wise its uses are limited to situations that are not easily
+described with CSS selectors.
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+Parsel also simplifies adding your own XPath extensions.
 
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
-
-Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``//node[1]`` selects all the nodes occurring first under their respective parents.
-
-``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text="""
-    ....:     <ul class="list">
-    ....:         <li>1</li>
-    ....:         <li>2</li>
-    ....:         <li>3</li>
-    ....:     </ul>
-    ....:     <ul class="list">
-    ....:         <li>4</li>
-    ....:         <li>5</li>
-    ....:         <li>6</li>
-    ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).getall()
-
-This gets all first ``<li>``  elements under whatever it is its parent::
-
-    >>> xp("//li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element in the whole document::
-
-    >>> xp("(//li)[1]")
-    ['<li>1</li>']
-
-This gets all first ``<li>``  elements under an ``<ul>``  parent::
-
-    >>> xp("//ul/li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
-
-    >>> xp("(//ul/li)[1]")
-    ['<li>1</li>']
-
-When querying by class, consider using CSS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Because an element can contain multiple CSS classes, the XPath way to select elements
-by class is the rather verbose::
-
-    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
-
-If you use ``@class='someclass'`` you may end up missing elements that have
-other classes, and if you just use ``contains(@class, 'someclass')`` to make up
-for that you may end up with more elements that you want, if they have a different
-class name that shares the string ``someclass``.
-
-As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').getall()
-    ['2014-07-23 19:00']
-
-This is cleaner than using the verbose XPath trick shown above. Just remember
-to use the ``.`` in the XPath expressions that will follow.
-
-.. _old-extraction-api:
-
-extract() and extract_first()
------------------------------
-
-If you're a long-time Scrapy user, you're probably familiar
-with ``.extract()`` and ``.extract_first()`` selector methods. These methods
-are still supported by Scrapy, there are no plans to deprecate them.
-
-However, Scrapy usage docs are now written using ``.get()`` and
-``.getall()`` methods. We feel that these new methods result in a more concise
-and readable code.
-
-The following examples show how these methods map to each other.
-
-1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
-
-     >>> response.css('a::attr(href)').get()
-     'image1.html'
-     >>> response.css('a::attr(href)').extract_first()
-     'image1.html'
-
-2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
-
-     >>> response.css('a::attr(href)').getall()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-     >>> response.css('a::attr(href)').extract()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
-
-     >>> response.css('a::attr(href)')[0].get()
-     'image1.html'
-     >>> response.css('a::attr(href)')[0].extract()
-     'image1.html'
-
-4. For consistency, there is also ``Selector.getall()``, which returns a list::
-
-    >>> response.css('a::attr(href)')[0].getall()
-    ['image1.html']
-
-So, the main difference is that output of ``.get()`` and ``.getall()`` methods
-is more predictable: ``.get()`` always returns a single result, ``.getall()``
-always returns a list of all extracted results. With ``.extract()`` method
-it was not always obvious if a result is a list or not; to get a single
-result either ``.extract()`` or ``.extract_first()`` should be called.
+.. autofunction:: parsel.xpathfuncs.set_xpathfunc
 
 
 .. _topics-selectors-ref:
@@ -909,6 +1013,11 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples:
+
+Examples
+========
+
 .. _selector-examples-html:
 
 Selector examples on HTML response
@@ -958,65 +1067,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
       sel.xpath("//g:price").getall()
 
-.. _removing-namespaces:
-
-Removing namespaces
--------------------
-
-When dealing with scraping projects, it is often quite convenient to get rid of
-namespaces altogether and just work with element names, to write more
-simple/convenient XPaths. You can use the
-:meth:`Selector.remove_namespaces` method for that.
-
-Let's show an example that illustrates this with GitHub blog atom feed.
-
-.. highlight:: sh
-
-First, we open the shell with the url we want to scrape::
-
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
-
-This is how the file starts::
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
-      ...
-
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
-
-.. highlight:: python
-
-Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
-
-    >>> response.xpath("//link")
-    []
-
-But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
-
-    >>> response.selector.remove_namespaces()
-    >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     ...
-
-If you wonder why the namespace removal procedure isn't always called by default
-instead of having to call it manually, this is because of two reasons, which, in order
-of relevance, are:
-
-1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform by default
-   for all documents crawled by Scrapy
-
-2. There could be some cases where using namespaces is actually required, in
-   case some element names clash between namespaces. These cases are very rare
-   though.
-
 .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799

From dc95ecbe25a5921f902fbccc93d50f7682cefda2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 12 Sep 2018 18:36:25 +0500
Subject: [PATCH 32/44] DOC use autodocs for selectors; document more methods
 and attributes; suggest get/getall

---
 docs/topics/selectors.rst  | 114 +++++++++----------------------------
 scrapy/selector/unified.py |  33 +++++++++++
 2 files changed, 60 insertions(+), 87 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 2db982b37..95d6a1a04 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -891,127 +891,67 @@ Built-in Selectors reference
 Selector objects
 ----------------
 
-.. class:: Selector(response=None, text=None, type=None)
+.. autoclass:: Selector
 
-  An instance of :class:`Selector` is a wrapper over response to select
-  certain parts of its content.
-
-  ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
-  :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
-  extracting data.
-
-  ``text`` is a unicode string or utf-8 encoded text for cases when a
-  ``response`` isn't available. Using ``text`` and ``response`` together is
-  undefined behavior.
-
-  ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
-
-    If ``type`` is ``None``, the selector automatically chooses the best type
-    based on ``response`` type (see below), or defaults to ``"html"`` in case it
-    is used together with ``text``.
-
-    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follows:
-
-        * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
-        * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
-        * ``"html"`` for anything else
-
-   Otherwise, if ``type`` is set, the selector type will be forced and no
-   detection will occur.
-
-  .. method:: xpath(query)
-
-      Find nodes matching the xpath ``query`` and return the result as a
-      :class:`SelectorList` instance with all elements flattened. List
-      elements implement :class:`Selector` interface too.
-
-      ``query`` is a string containing the XPATH query to apply.
+  .. automethod:: xpath
 
       .. note::
 
           For convenience, this method can be called as ``response.xpath()``
 
-  .. method:: css(query)
-
-      Apply the given CSS selector and return a :class:`SelectorList` instance.
-
-      ``query`` is a string containing the CSS selector to apply.
-
-      In the background, CSS queries are translated into XPath queries using
-      `cssselect`_ library and run ``.xpath()`` method.
+  .. automethod:: css
 
       .. note::
 
-          For convenience this method can be called as ``response.css()``
+          For convenience, this method can be called as ``response.css()``
 
-  .. method:: extract()
+  .. automethod:: get
 
-     Serialize and return the matched nodes as a list of unicode strings.
-     Percent encoded content is unquoted.
+     See also: :ref:`old-extraction-api`
 
-  .. method:: re(regex)
+  .. autoattribute:: attrib
 
-     Apply the given regex and return a list of unicode strings with the
-     matches.
+     See also: :ref:`selecting-attributes`.
 
-     ``regex`` can be either a compiled regular expression or a string which
-     will be compiled to a regular expression using ``re.compile(regex)``
+  .. automethod:: re
 
-    .. note::
+  .. automethod:: re_first
 
-        Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``&lt;`` and ``&amp;``).
+  .. automethod:: register_namespace
 
-  .. method:: register_namespace(prefix, uri)
+  .. automethod:: remove_namespaces
 
-     Register the given namespace to be used in this :class:`Selector`.
-     Without registering namespaces you can't select or extract data from
-     non-standard namespaces. See examples below.
+  .. automethod:: __bool__
 
-  .. method:: remove_namespaces()
-
-     Remove all namespaces, allowing to traverse the document using
-     namespace-less xpaths. See example below.
-
-  .. method:: __nonzero__()
-
-     Returns ``True`` if there is any real content selected or ``False``
-     otherwise.  In other words, the boolean value of a :class:`Selector` is
-     given by the contents it selects.
+  .. automethod:: getall
 
+     This method is added to Selector for consistency; it is more useful
+     with SelectorList. See also: :ref:`old-extraction-api`
 
 SelectorList objects
 --------------------
 
-.. class:: SelectorList
+.. autoclass:: SelectorList
 
-   The :class:`SelectorList` class is a subclass of the builtin ``list``
-   class, which provides a few additional methods.
+   .. automethod:: xpath
 
-   .. method:: xpath(query)
+   .. automethod:: css
 
-       Call the ``.xpath()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+   .. automethod:: getall
 
-       ``query`` is the same argument as the one in :meth:`Selector.xpath`
+      See also: :ref:`old-extraction-api`
 
-   .. method:: css(query)
+   .. automethod:: get
 
-       Call the ``.css()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+      See also: :ref:`old-extraction-api`
 
-       ``query`` is the same argument as the one in :meth:`Selector.css`
+   .. automethod:: re
 
-   .. method:: extract()
+   .. automethod:: re_first
 
-       Call the ``.extract()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
-
-   .. method:: re()
-
-       Call the ``.re()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
+   .. autoattribute:: attrib
 
+      See also: :ref:`selecting-attributes`.
 
 .. _selector-examples:
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 64cb0232c..8f6cb1d79 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -27,6 +27,10 @@ def _response_from_text(text, st):
 
 
 class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
+    """
+    The :class:`SelectorList` class is a subclass of the builtin ``list``
+    class, which provides a few additional methods.
+    """
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]
@@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
 
 
 class Selector(_ParselSelector, object_ref):
+    """
+    An instance of :class:`Selector` is a wrapper over response to select
+    certain parts of its content.
+
+    ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
+    :class:`~scrapy.http.XmlResponse` object that will be used for selecting
+    and extracting data.
+
+    ``text`` is a unicode string or utf-8 encoded text for cases when a
+    ``response`` isn't available. Using ``text`` and ``response`` together is
+    undefined behavior.
+
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+    or ``None`` (default).
+
+    If ``type`` is ``None``, the selector automatically chooses the best type
+    based on ``response`` type (see below), or defaults to ``"html"`` in case it
+    is used together with ``text``.
+
+    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
+    inferred from the response type as follows:
+
+    * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
+    * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+    * ``"html"`` for anything else
+
+    Otherwise, if ``type`` is set, the selector type will be forced and no
+    detection will occur.
+    """
 
     __slots__ = ['response']
     selectorlist_cls = SelectorList

From 9db21e55028feec932136e74dad0c0f0dbe7f436 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 15 Sep 2018 02:43:37 +0500
Subject: [PATCH 33/44] DOC fix remove_namespaces example

See https://github.com/scrapy/parsel/pull/119
---
 docs/topics/selectors.rst | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 95d6a1a04..20c3fff3c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -643,27 +643,30 @@ namespaces altogether and just work with element names, to write more
 simple/convenient XPaths. You can use the
 :meth:`Selector.remove_namespaces` method for that.
 
-Let's show an example that illustrates this with GitHub blog atom feed.
+Let's show an example that illustrates this with the Python Insider blog atom feed.
 
 .. highlight:: sh
 
 First, we open the shell with the url we want to scrape::
 
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
+    $ scrapy shell https://feeds.feedburner.com/PythonInsider
 
 This is how the file starts::
 
     <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
+    <?xml-stylesheet ...
+    <feed xmlns="http://www.w3.org/2005/Atom"
+          xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/"
+          xmlns:blogger="http://schemas.google.com/blogger/2008"
+          xmlns:georss="http://www.georss.org/georss"
+          xmlns:gd="http://schemas.google.com/g/2005"
+          xmlns:thr="http://purl.org/syndication/thread/1.0"
+          xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
       ...
 
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+You can see several namespace declarations including a default
+"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for
+"http://schemas.google.com/g/2005".
 
 .. highlight:: python
 
@@ -678,8 +681,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+     <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default

From 2c3b2158c99953823500f275ccc5206c11c9a811 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:02:17 +0500
Subject: [PATCH 34/44] DOC address @stummjr's review comments

* fixed several small issues
* re-written "Creating Selectors" section
* fixed remaining .extract usage in tests
---
 docs/intro/tutorial.rst       |  6 ++--
 docs/topics/selectors.rst     | 52 +++++++++++++++++++----------------
 tests/test_utils_iterators.py |  4 +--
 3 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 46e84b21c..ad17ef096 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -285,9 +285,9 @@ There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
-Besides the :meth:`~scrapy.selector.Selector.extract` and
+Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
 expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
@@ -649,7 +649,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).get().strip()
+                return response.css(query).get(default='').strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 20c3fff3c..68913c697 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -50,28 +50,8 @@ Constructing selectors
 
 .. highlight:: python
 
-Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing **text** or :class:`~scrapy.http.TextResponse`
-object. It automatically chooses the best parsing rules (XML vs HTML) based on
-input type::
-
-    >>> from scrapy.selector import Selector
-    >>> from scrapy.http import HtmlResponse
-
-Constructing from text::
-
-    >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').get()
-    'good'
-
-Constructing from response::
-
-    >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').get()
-    'good'
-
-For convenience, response objects expose a selector on `.selector` attribute.
-By using it you can ensure the response body is parsed only once::
+Response objects expose a :class:`~scrapy.selector.Selector` instance
+on ``.selector`` attribute::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -84,10 +64,34 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
+Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
+constructed by passing either :class:`~scrapy.http.TextResponse` object or
+markup as an unicode string (in ``text`` argument).
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
-shortcuts.
+shortcuts. By using ``response.selector`` or one of these shortcuts
+you can also ensure the response body is parsed only once.
+
+But if required, it is possible to use ``Selector`` directly.
+Constructing from text::
+
+    >>> from scrapy.selector import Selector
+    >>> body = '<html><body><span>good</span></body></html>'
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
+
+Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
+:class:`~scrapy.http.TextResponse` subclasses::
+
+    >>> from scrapy.selector import Selector
+    >>> from scrapy.http import HtmlResponse
+    >>> response = HtmlResponse(url='http://example.com', body=body)
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
+
+``Selector`` automatically chooses the best parsing rules
+(XML vs HTML) based on input type.
 
 Using selectors
 ---------------
@@ -139,7 +143,7 @@ is returned. ``.getall()`` returns a list with all results.
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 
-    >>> selector.css('title::text').get()
+    >>> response.css('title::text').get()
     'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 00eb78068..2d845697e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -32,8 +32,8 @@ class XmliterTestCase(unittest.TestCase):
         for x in self.xmliter(response, 'product'):
             attrs.append((
                 x.attrib['id'],
-                x.xpath("name/text()").extract(),
-                x.xpath("./type/text()").extract()))
+                x.xpath("name/text()").getall(),
+                x.xpath("./type/text()").getall()))
 
         self.assertEqual(attrs,
                          [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])

From ffbd33edac0367e9f975b9863e0c31e1c2b72ebc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:03:35 +0500
Subject: [PATCH 35/44] DOC mention gotcha with `foo::text` selector and empty
 `foo` elements

also, move "Selecting attributes" reference closer to `a::atr(href)` example
---
 docs/topics/selectors.rst | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 68913c697..9dced7473 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -279,6 +279,19 @@ Examples:
      'Name: My image 5 ',
      '\n  ']
 
+* ``foo::text`` returns no results if ``foo`` element exists, but contains
+  no text (i.e. text is empty)::
+
+    >>> response.css('img::text').getall()
+    []
+
+  This means ``.css('foo::text').get()`` could return None even if an element
+  exists. Use ``default=''`` if you always want a string::
+
+    >>> response.css('img::text').get()
+    >>> response.css('img::text').get(default='')
+    ''
+
 * ``a::attr(href)`` selects the *href* attribute value of descendant links::
 
     >>> response.css('a::attr(href)').getall()
@@ -288,15 +301,14 @@ Examples:
      'image4.html',
      'image5.html']
 
+.. note::
+    See also: :ref:`selecting-attributes`.
+
 .. note::
     You cannot chain these pseudo-elements. But in practice it would not
     make much sense: text nodes do not have attributes, and attribute values
     are string values already and do not have children nodes.
 
-.. note::
-    See also: :ref:`selecting-attributes`.
-
-
 .. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors:

From edaf74bfaeef7d995676ad8f6bfb8056a8e6966d Mon Sep 17 00:00:00 2001
From: jfflisikowski <jfflisikowski@gmail.com>
Date: Tue, 2 Oct 2018 19:48:48 +0200
Subject: [PATCH 36/44] Correct the unclear comments by adding <# < processing
 code not shown >

---
 docs/topics/debug.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index d1991c02f..f93aa2c72 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -18,11 +18,13 @@ Consider the following scrapy spider below::
             )
 
         def parse(self, response):
-            # collect `item_urls`
+            # <processing code not shown>
+            # collect `item_urls` 
             for item_url in item_urls:
                 yield scrapy.Request(item_url, self.parse_item)
 
         def parse_item(self, response):
+            # <processing code not shown>
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url

From c9b5bd6ad7728274b0f82fc3211c5ade5cd0d389 Mon Sep 17 00:00:00 2001
From: Immanuella Lim <immanuella@live.com>
Date: Thu, 18 Oct 2018 02:22:32 +0800
Subject: [PATCH 37/44] Remove ad link Dive Into Python3 from tutorial docs

---
 docs/intro/tutorial.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ad17ef096..143e018ac 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -23,7 +23,7 @@ start by getting an idea of what the language is like, to get the most out of
 Scrapy.
 
 If you're already familiar with other languages, and want to learn Python
-quickly, we recommend reading through `Dive Into Python 3`_.  Alternatively,
+quickly, we recommend reading through `Crash into Python`_.  Alternatively,
 you can follow the `Python Tutorial`_.
 
 If you're new to programming and want to start with Python, the following books
@@ -40,7 +40,7 @@ as well as the `suggested resources in the learnpython-subreddit`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Dive Into Python 3: http://www.diveintopython3.net
+.. _Crash into Python: https://stephensugden.com/crash_into_python/
 .. _Python Tutorial: https://docs.python.org/3/tutorial
 .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
 .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/

From 6c98010f110c432a2311c1aef1d463dc5a6ccba4 Mon Sep 17 00:00:00 2001
From: Immanuella Lim <immanuella@live.com>
Date: Sun, 4 Nov 2018 16:04:45 +0800
Subject: [PATCH 38/44] Remove 'Dive into Python3' reference

---
 docs/intro/tutorial.rst | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 143e018ac..41e61542a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -22,9 +22,7 @@ Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
 Scrapy.
 
-If you're already familiar with other languages, and want to learn Python
-quickly, we recommend reading through `Crash into Python`_.  Alternatively,
-you can follow the `Python Tutorial`_.
+If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
 
 If you're new to programming and want to start with Python, the following books
 may be useful to you: 
@@ -40,7 +38,6 @@ as well as the `suggested resources in the learnpython-subreddit`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Crash into Python: https://stephensugden.com/crash_into_python/
 .. _Python Tutorial: https://docs.python.org/3/tutorial
 .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
 .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/

From 491929c212999aa816e561aeed19a902664d01e5 Mon Sep 17 00:00:00 2001
From: Todd <toddrme2178@gmail.com>
Date: Fri, 16 Nov 2018 13:38:19 -0500
Subject: [PATCH 39/44] Include additional files in sdists

In particular this includes files needed for running the tests, as well as the changelog.
---
 MANIFEST.in | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 94de4f3bf..ae7db51fa 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,13 +3,24 @@ include AUTHORS
 include INSTALL
 include LICENSE
 include MANIFEST.in
+include NEWS
+
 include scrapy/VERSION
 include scrapy/mime.types
+
+include codecov.yml
+include conftest.py
+include pytest.ini
+include requirements-*.txt
+include tox.ini
+
 recursive-include scrapy/templates *
 recursive-include scrapy license.txt
 recursive-include docs *
 prune docs/build
+
 recursive-include extras *
 recursive-include bin *
 recursive-include tests *
+
 global-exclude __pycache__ *.py[cod]

From 274b65dff4dc8b8300d872171679f173fbe0a746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 16:36:05 +0100
Subject: [PATCH 40/44] Add a troubleshooting section to the installation
 instructions

Its initial content covers the workaround for #2473.
---
 docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4a9aa3cfb..daec7fcb7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the
 We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
 to avoid conflicting with your system packages.
 
-For more detailed and platform specifics instructions, read on.
+For more detailed and platform specifics instructions, as well as
+troubleshooting information, read on.
 
 
 Things that are good to know
@@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency.
 To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
 
 
+.. _intro-install-troubleshooting:
+
+Troubleshooting
+===============
+
+AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+----------------------------------------------------------------
+
+After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
+exception with the following traceback::
+
+    […]
+      File "[…]/site-packages/twisted/protocols/tls.py", line 63, in <module>
+        from twisted.internet._sslverify import _setAcceptableProtocols
+      File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in <module>
+        TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
+    AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+
+The reason you get this exception is that your system or virtual environment
+has a version of pyOpenSSL that your version of Twisted does not support.
+
+To install a version of pyOpenSSL that your version of Twisted supports,
+reinstall Twisted with the :code:`tls` extra option::
+
+    pip install twisted[tls]
+
+For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
+
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
 .. _lxml: http://lxml.de/

From 62f3349c1aee54599ab7ee8755d2b31090639105 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 17:14:10 +0100
Subject: [PATCH 41/44] Document the SCRAPY_PROJECT environment variable

Fixes #1109
---
 docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index ef9c45196..97f8311de 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
 * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
-* ``SCRAPY_PROJECT``
+* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
 * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
@@ -71,6 +71,33 @@ the project settings. Here is an example::
     [settings]
     default = myproject.settings
 
+.. _topics-project-envvar:
+
+Sharing the root directory between projects
+===========================================
+
+A project root directory, the one that contains the ``scrapy.cfg``, may be
+shared by multiple Scrapy projects, each with its own settings module.
+
+In that case, you must define one or more aliases for those settings modules
+under ``[settings]`` in your ``scrapy.cfg`` file::
+
+    [settings]
+    default = myproject1.settings
+    project1 = myproject1.settings
+    project2 = myproject2.settings
+
+By default, the ``scrapy`` command-line tool will use the ``default`` settings.
+Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
+for ``scrapy`` to use::
+
+    $ scrapy settings --get BOT_NAME
+    Project 1 Bot
+    $ export SCRAPY_PROJECT=project2
+    $ scrapy settings --get BOT_NAME
+    Project 2 Bot
+
+
 Using the ``scrapy`` tool
 =========================
 

From 4d48759978ac2405bc2cb30f84af948693e4cad3 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Mon, 10 Dec 2018 14:44:15 +0800
Subject: [PATCH 42/44] remove "sudo: false" now that travis no longer supports
 it

https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4218d13bf..08b0bf119 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,4 @@
 language: python
-sudo: false
 branches:
   only:
     - master

From f6dfc5f3dd56b7c823e3f53f7f9f63515ca7c3e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:23:23 -0300
Subject: [PATCH 43/44] Fix boto import error under Jessie testing environment

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 4218d13bf..252c783d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,6 +43,11 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
+      if [ "$TOXENV" = "jessie" ]; then
+        # Not used directly but allows boto GCE plugins to load.
+        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+        pip install google-compute-engine
+      fi
   - pip install -U tox twine wheel codecov
 
 script: tox

From 8ed6beb7f9199e8924cd03bd34a46194c3d82e32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:39:29 -0300
Subject: [PATCH 44/44] Needs to be installed within tox env

---
 .travis.yml | 5 -----
 tox.ini     | 3 +++
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 252c783d7..4218d13bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,11 +43,6 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
-      if [ "$TOXENV" = "jessie" ]; then
-        # Not used directly but allows boto GCE plugins to load.
-        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-        pip install google-compute-engine
-      fi
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index e5543fe2a..0c0f8f7b7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,9 @@ deps =
     cssselect==0.9.1
     zope.interface==4.1.1
     -rtests/requirements-py2.txt
+# Not used directly but allows boto GCE plugins to load.
+# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+    google-compute-engine==2.8.12
 
 [testenv:trunk]
 basepython = python2.7